{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym, random, pickle, os.path, math, glob\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "import pdb\n",
    "\n",
    "from atari_wrappers import make_atari, wrap_deepmind\n",
    "from IPython.display import clear_output\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x18f3ad9c388>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAO+UlEQVR4nO3da4xc5X3H8e/f67vB+FJAjo1iiBCBJMKmFoFSVQRCQ0kKUUsqUFRFFRJv0haaSAHaF1GkSiVSlZAXUVQLkloVJYADDbIiqGXgRaXKYC4hgO1gYwILDjYGh2Cwsdf/vjjH9sqdZc/uXHbGz/cjrWbOZfY8Z49+85w5e+b5R2Yi6cQ3baobIKk3DLtUCMMuFcKwS4Uw7FIhDLtUiLbCHhFXRsTWiNgWEbd2qlGSOi8m+3/2iBgCfg1cAQwDTwLXZ+aLnWuepE6Z3sZrLwS2ZebLABHxU+AaYMywz4xZOZt54//mubOPPZ8WbTSxc7Jux6E57bVn+v7qzTVGvJmp20o8Zvv37+XDg/ta7nA7YV8KvDZqehj47Ee9YDbz+GxcPu4vjvM+dfT5yEkzJ9m8zhqZNQTA7vPba8/iFw4CMGPfobbbpI925Ji99Zn2jtmizYNzzJ7c9MMxl7UT9lbvHv/vrS8ibgRuBJjN3DY2J6kd7YR9GDhj1PQy4I3jV8rM1cBqgPknL83Dq1a2scmpc3ioem/b9/GRtn7Pgu3VNdEZ+9puksZx5Ji9t7y9Y3bKjhPjmLVzNf5J4OyIODMiZgLXAQ91plmSOm3SPXtmHoqIvwUeAYaAH2fmCx1rWZ+Z8X71ee0T9zd/zY4/nwXA4dn9f2GnNJ9Y+2HL+a98sbo4PDLncC+b0xPtnMaTmb8AftGhtkjqIu+gkwrRVs9eksND1fvi785s/m+cHPL0Xf3Dnl0qhD17QyOzqvfFt1eeeBduVAZ7dqkQhl0qhKfxHbD00ebrztzb//dX68Rkzy4VwrBLhfA0XkXav7j1/RI57cS9N8KeXSqEPXsHvH5Z83WXPlr9yWfvaf1FDPXG658ba4k9u6QBZ9ilQnga39DQgeo22UXPtPcnm77vYCeaowaOHrNnPWZgzy4Vo6c9+4GF09h+bX+MFjt57X0R5u0VQ/WzoY9cT51UzjE7sH3s/nvcnj0ifhwRuyLi+VHzFkXE+oh4qX5c2KG2SuqSJqfx/w5cedy8W4ENmXk2sKGeltTHGpV/iojlwLrM/HQ9vRW4NDN3RsQS4PHMPGe837Pq/Nn5xCNnjLeapEm68AuvsemX+1tWhJnsBbrTM3MnQP142mQbJ6k3un41PiJujIhNEbFp9572BuuXNHmTDfub9ek79eOusVbMzNWZuSozV526uP+vZkonqsmG/SHga/XzrwE/70xzJHVLk3+93QP8L3BORAxHxA3A7cAVEfESVX3227vbTEntGvemmsy8foxF49deltQ3vF1WKkRPb5fd8sECLnnuL3q5SakoWz5YM+Yye3apED3t2aftHGLuv5zSy01KRZm2c+x/b9uzS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUCMMuFcKwS4VoMizVGRHxWERsjogXIuKmer5VYaQB0qRnPwR8MzPPBS4Cvh4R52FVGGmgjBv2zNyZmU/Xz38PbAaWAtcAR4bFWAN8uVuNlNS+CX1mr8tArQQ20rAqzOgiER8e3NdeayVNWuOwR8RJwM+AmzPz3aavG10kYuaMeZNpo6QOaBT2iJhBFfS7M/OBenbjqjCSpl6Tq/EB3AVszszvjVpkVRhpgDQZcPIS4K+BX0XEs/W8f6SqAnNfXSHmVeAr3WmipE5oUhHmf4CW9Z6xKow0MLyDTiqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSpEkzHoZkfEExHxy7oizHfq+WdGxMa6Isy9ETGz+82VNFlNevYDwGWZeT6wArgyIi4Cvgt8v64I8w5wQ/eaKaldTSrCZGa+V0/OqH8SuAxYW8+3IozU55qOGz9Ujyy7C1gPbAf2ZuahepVhqpJQrV5rRRipDzQKe2aOZOYKYBlwIXBuq9XGeK0VYaQ+MKGr8Zm5F3icqprrgog4MhT1MuCNzjZNUic1uRp/akQsqJ/PAT5PVcn1MeDaejUrwkh9rklFmCXAmogYonpzuC8z10XEi8BPI+KfgWeoSkRJ6lNNKsI8R1Wm+fj5L1N9fpc0ALyDTiqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRCGXSpE47DXw0k/ExHr6mkrwkgDZCI9+01UA00eYUUYaYA0LRKxDPgicGc9HVgRRhooTXv2O4BvAYfr6cVYEUYaKE3Gjf8SsCsznxo9u8WqVoSR+liTceMvAa6OiKuA2cB8qp5+QURMr3t3K8JIfa5JFdfbMnNZZi4HrgMezcyvYkUYaaC083/2W4BvRMQ2qs/wVoSR+liT0/ijMvNxqsKOVoSRBox30EmFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFaDRSTUS8AvweGAEOZeaqiFgE3AssB14B/ioz3+lOMyW1ayI9++cyc0VmrqqnbwU21BVhNtTTkvrUhMagO841wKX18zVUY9Pd0mZ7pBPKe0tnHX2+b8mxvnX2nqrMwik79vesLU179gT+OyKeiogb63mnZ+ZOgPrxtFYvtCKM1B+a9uyXZOYbEXEasD4itjTdQGauBlYDzD95acuqMdKJav/CY/3pe8tHRi0ZAuCUHb1rS6OePTPfqB93AQ9SDSH9ZkQsAagfd3WrkZLa16TW27yIOPnIc+BPgeeBh6gqwYAVYaS+1+Q0/nTgwapKM9OB/8zMhyPiSeC+iLgBeBX4SveaKald44a9rvxyfov5e4DLu9EoSZ3nHXRSIQy7VAjDLhXCsEuFMOxSIQy7VAjDLhXCsEuFaOcrrpLGMbT/2He/Zuw91rcOfdD7ttizS4Uw7FIhPI2Xumjhtv2jnk9hQ7Bnl4ph2KVCGHapEIZdKoRhlwrRKOwRsSAi1kbElojYHBEXR8SiiFgfES/Vjwu73VhJk9e0Z/8B8HBmfpJqiKrNWBFGGihNRpedD/wJcBdAZn6YmXupKsKsqVdbA3y5W42U1L4mPftZwG7gJxHxTETcWQ8pbUUYaYA0Cft04ALgR5m5EtjHBE7ZM3N1Zq7KzFUzZ8ybZDMltatJ2IeB4czcWE+vpQq/FWGkATJu2DPzt8BrEXFOPety4EWsCCMNlKZfhPk74O6ImAm8DPwN1RuFFWGkAdEo7Jn5LLCqxSIrwkgDwjvopEIYdqkQhl0qhGGXCmHYpUIYdqkQAz/g5FufmgPA+x87Nj73yTuOLR894J9UMnt2qRAD37Mfqr9bc3DB4aPzRuYMTVFrpP5lzy4VwrBLhTDsUiEMu1QIwy4VwrBLhTDsUiGaDCV9TkQ8O+rn3Yi42SIR0mBpMgbd1sxckZkrgD8E3gcexCIR0kCZ6Gn85cD2zPwNFomQBspEw34dcE/9vFGRCEn9oXHY65Flrwbun8gGrAgj9YeJ9Ox/BjydmW/W042KRFgRRuoPEwn79Rw7hQeLREgDpWl99rnAFcADo2bfDlwRES/Vy27vfPMkdUrTIhHvA4uPm7eHPigSMf831ffYZ7197H1rzjsjU9UcqW95B51UiIEfqWbumwfqxyluiNTn7NmlQhh2qRCGXSqEYZcKYdilQhh2qRCGXSqEYZcKYdilQhh2qRA9vV32wMJpbL92Zi83KRXlwPax+297dqkQPe3ZP7NwN0/85b/1cpNSUS5cvXvMZfbsUiEMu1SIpsNS/UNEvBARz0fEPRExOyLOjIiNdUWYe+vRZyX1qSbln5YCfw+sysxPA0NU48d/F/h+XRHmHeCGbjZUUnuansZPB+ZExHRgLrATuAxYWy+3IozU55rUensd+FfgVaqQ/w54CtibmYfq1YaBpd1qpKT2NTmNX0hV1+1M4GPAPKqCEcfLMV5/tCLM7j2O+ipNlSan8Z8HdmTm7sw8SDV2/B8BC+rTeoBlwButXjy6Isypi4c60mhJE9ck7K8CF0XE3IgIqrHiXwQeA66t17EijNTnmnxm30h1Ie5p4Ff1a1YDtwDfiIhtVAUk7upiOyW1qWlFmG8D3z5u9svAhR1vkaSu8A46qRCGXSqEYZcKYdilQkRmy3thurOxiN3APuCtnm20+/4A96dfnUj7As325+OZeWqrBT0NO0BEbMrMVT3daBe5P/3rRNoXaH9/PI2XCmHYpUJMRdhXT8E2u8n96V8n0r5Am/vT88/skqaGp/FSIXoa9oi4MiK2RsS2iLi1l9tuV0ScERGPRcTmejy+m+r5iyJifT0W3/r6+/8DIyKGIuKZiFhXTw/s2IIRsSAi1kbElvo4XTzIx6fTYz/2LOwRMQT8kGrgi/OA6yPivF5tvwMOAd/MzHOBi4Cv1+2/FdhQj8W3oZ4eJDcBm0dND/LYgj8AHs7MTwLnU+3XQB6froz9mJk9+QEuBh4ZNX0bcFuvtt+F/fk5cAWwFVhSz1sCbJ3qtk1gH5ZRBeAyYB0QVDdtTG91zPr5B5gP7KC+DjVq/kAeH6ph3l4DFlF9O3Ud8IV2jk8vT+OPNP6IgR23LiKWAyuBjcDpmbkToH48bepaNmF3AN8CDtfTixncsQXPAnYDP6k/ltwZEfMY0OOTXRj7sZdhjxbzBu5fARFxEvAz4ObMfHeq2zNZEfElYFdmPjV6dotVB+UYTQcuAH6UmSupbsseiFP2Vtod+7GVXoZ9GDhj1PSY49b1q4iYQRX0uzPzgXr2mxGxpF6+BNg1Ve2boEuAqyPiFeCnVKfyd9BwbME+NAwMZzWyElSjK13A4B6ftsZ+bKWXYX8SOLu+mjiT6mLDQz3cflvq8ffuAjZn5vdGLXqIagw+GKCx+DLztsxclpnLqY7Fo5n5VQZ0bMHM/C3wWkScU886MlbiQB4fujH2Y48vOlwF/BrYDvzTVF8EmWDb/5jqlOk54Nn65yqqz7kbgJfqx0VT3dZJ7NulwLr6+VnAE8A24H5g1lS3bwL7sQLYVB+j/wIWDvLxAb4DbAGeB/4DmNXO8fEOOqkQ3kEnFcKwS4Uw7FIhDLtUCMMuFcKwS4Uw7FIhDLtUiP8DCDlT5Y7FnrkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create and wrap the environment\n",
    "env = make_atari('PongNoFrameskip-v4') # only use in no frameskip environment\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True )\n",
    "n_actions = env.action_space.n\n",
    "state_dim = env.observation_space.shape\n",
    "\n",
    "# env.render()\n",
    "test = env.reset()\n",
    "for i in range(100):\n",
    "    test = env.step(env.action_space.sample())[0]\n",
    "\n",
    "plt.imshow(test._force()[...,0])\n",
    "\n",
    "#plt.imshow(env.render(\"rgb_array\"))\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "        self.fc5 = nn.Linear(512, num_actions)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        return self.fc5(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SumTree\n",
    "# a binary tree data structure where the parent’s value is the sum of its children\n",
    "class SumTree:\n",
    "    write = 0\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.tree = np.zeros(2 * capacity - 1)\n",
    "        self.data = np.zeros(capacity, dtype=object)\n",
    "        self.n_entries = 0\n",
    "\n",
    "    # update to the root node\n",
    "    def _propagate(self, idx, change):\n",
    "        parent = (idx - 1) // 2\n",
    "\n",
    "        self.tree[parent] += change\n",
    "\n",
    "        if parent != 0:\n",
    "            self._propagate(parent, change)\n",
    "\n",
    "    # find sample on leaf node\n",
    "    def _retrieve(self, idx, s):\n",
    "        left = 2 * idx + 1\n",
    "        right = left + 1\n",
    "\n",
    "        if left >= len(self.tree):\n",
    "            return idx\n",
    "\n",
    "        if s <= self.tree[left]:\n",
    "            return self._retrieve(left, s)\n",
    "        else:\n",
    "            return self._retrieve(right, s - self.tree[left])\n",
    "\n",
    "    def total(self):\n",
    "        return self.tree[0]\n",
    "\n",
    "    # store priority and sample\n",
    "    def add(self, p, data):\n",
    "        idx = self.write + self.capacity - 1\n",
    "\n",
    "        self.data[self.write] = data\n",
    "        self.update(idx, p)\n",
    "\n",
    "        self.write += 1\n",
    "        if self.write >= self.capacity:\n",
    "            self.write = 0\n",
    "\n",
    "        if self.n_entries < self.capacity:\n",
    "            self.n_entries += 1\n",
    "\n",
    "    # update priority\n",
    "    def update(self, idx, p):\n",
    "        change = p - self.tree[idx]\n",
    "\n",
    "        self.tree[idx] = p\n",
    "        self._propagate(idx, change)\n",
    "\n",
    "    # get priority and sample\n",
    "    def get(self, s):\n",
    "        idx = self._retrieve(0, s)\n",
    "        dataIdx = idx - self.capacity + 1\n",
    "\n",
    "        return (idx, self.tree[idx], self.data[dataIdx])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory_Buffer_PER(object):\n",
    "    # stored as ( s, a, r, s_ ) in SumTree\n",
    "    def __init__(self, memory_size=1000, a = 0.6, e = 0.01):\n",
    "        self.tree =  SumTree(memory_size)\n",
    "        self.memory_size = memory_size\n",
    "        self.prio_max = 0.1\n",
    "        self.a = a\n",
    "        self.e = e\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        data = (state, action, reward, next_state, done)\n",
    "        p = (np.abs(self.prio_max) + self.e) ** self.a #  proportional priority\n",
    "        self.tree.add(p, data)\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        idxs = []\n",
    "        segment = self.tree.total() / batch_size\n",
    "        priorities = []\n",
    "\n",
    "        for i in range(batch_size):\n",
    "            a = segment * i\n",
    "            b = segment * (i + 1)\n",
    "            s = random.uniform(a, b)\n",
    "            idx, p, data = self.tree.get(s)\n",
    "            \n",
    "            state, action, reward, next_state, done= data\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(next_state)\n",
    "            dones.append(done)\n",
    "            priorities.append(p)\n",
    "            idxs.append(idx)\n",
    "        return idxs, np.concatenate(states), actions, rewards, np.concatenate(next_states), dones\n",
    "    \n",
    "    def update(self, idxs, errors):\n",
    "        self.prio_max = max(self.prio_max, max(np.abs(errors)))\n",
    "        for i, idx in enumerate(idxs):\n",
    "            p = (np.abs(errors[i]) + self.e) ** self.a\n",
    "            self.tree.update(idx, p) \n",
    "        \n",
    "    def size(self):\n",
    "        return self.tree.n_entries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN_PERAgent: \n",
    "    def __init__(self, in_channels = 1, action_space = None, USE_CUDA = False, memory_size = 10000, prio_a = 0.6, prio_e = 0.001, epsilon  = 1, lr = 1e-4):\n",
    "        self.epsilon = epsilon\n",
    "        self.action_space = action_space\n",
    "        self.memory_buffer = Memory_Buffer_PER(memory_size, a = prio_a, e = prio_e)\n",
    "        self.DQN = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target = DQN(in_channels = in_channels, num_actions = action_space.n)\n",
    "        self.DQN_target.load_state_dict(self.DQN.state_dict())\n",
    "\n",
    "\n",
    "        self.USE_CUDA = USE_CUDA\n",
    "        if USE_CUDA:\n",
    "            self.DQN = self.DQN.cuda()\n",
    "            self.DQN_target = self.DQN_target.cuda()\n",
    "        self.optimizer = optim.RMSprop(self.DQN.parameters(),lr=lr, eps=0.001, alpha=0.95)\n",
    "\n",
    "    def observe(self, lazyframe):\n",
    "        # from Lazy frame to tensor\n",
    "        state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()\n",
    "        if self.USE_CUDA:\n",
    "            state = state.cuda()\n",
    "        return state\n",
    "\n",
    "    def value(self, state):\n",
    "        q_values = self.DQN(state)\n",
    "        return q_values\n",
    "    \n",
    "    def act(self, state, epsilon = None):\n",
    "        \"\"\"\n",
    "        sample actions with epsilon-greedy policy\n",
    "        recap: with p = epsilon pick random action, else pick action with highest Q(s,a)\n",
    "        \"\"\"\n",
    "        if epsilon is None: epsilon = self.epsilon\n",
    "\n",
    "        q_values = self.value(state).cpu().detach().numpy()\n",
    "        if random.random()<epsilon:\n",
    "            aciton = random.randrange(self.action_space.n)\n",
    "        else:\n",
    "            aciton = q_values.argmax(1)[0]\n",
    "        return aciton\n",
    "    \n",
    "    def compute_td_loss(self,idxs, states, actions, rewards, next_states, is_done, gamma=0.99):\n",
    "        \"\"\" Compute td loss using torch operations only. Use the formula above. \"\"\"\n",
    "        actions = torch.tensor(actions).long()    # shape: [batch_size]\n",
    "        rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]\n",
    "        is_done = torch.tensor(is_done).bool()  # shape: [batch_size]\n",
    "        \n",
    "        if self.USE_CUDA:\n",
    "            actions = actions.cuda()\n",
    "            rewards = rewards.cuda()\n",
    "            is_done = is_done.cuda()\n",
    "\n",
    "        # get q-values for all actions in current states\n",
    "        predicted_qvalues = self.DQN(states)\n",
    "\n",
    "        # select q-values for chosen actions\n",
    "        predicted_qvalues_for_actions = predicted_qvalues[\n",
    "          range(states.shape[0]), actions\n",
    "        ]\n",
    "\n",
    "        # compute q-values for all actions in next states\n",
    "        predicted_next_qvalues = self.DQN_target(next_states) # YOUR CODE\n",
    "\n",
    "        # compute V*(next_states) using predicted next q-values\n",
    "        next_state_values =  predicted_next_qvalues.max(-1)[0] # YOUR CODE\n",
    "\n",
    "        # compute \"target q-values\" for loss - it's what's inside square parentheses in the above formula.\n",
    "        target_qvalues_for_actions = rewards + gamma *next_state_values # YOUR CODE\n",
    "\n",
    "        # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist\n",
    "        target_qvalues_for_actions = torch.where(\n",
    "            is_done, rewards, target_qvalues_for_actions)\n",
    "\n",
    "        # mean squared error loss to minimize\n",
    "        errors = (predicted_qvalues_for_actions - target_qvalues_for_actions).detach().cpu().squeeze().tolist()\n",
    "        self.memory_buffer.update(idxs, errors)\n",
    "        loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def sample_from_buffer(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        idxs = []\n",
    "        segment = self.memory_buffer.tree.total() / batch_size\n",
    "        priorities = []\n",
    "\n",
    "        for i in range(batch_size):\n",
    "            a = segment * i\n",
    "            b = segment * (i + 1)\n",
    "            s = random.uniform(a, b)\n",
    "            idx, p, data = self.memory_buffer.tree.get(s)\n",
    "            \n",
    "            frame, action, reward, next_frame, done= data\n",
    "            states.append(self.observe(frame))\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(self.observe(next_frame))\n",
    "            dones.append(done)\n",
    "            priorities.append(p)\n",
    "            idxs.append(idx)\n",
    "        return idxs, torch.cat(states), actions, rewards, torch.cat(next_states), dones\n",
    "\n",
    "    def learn_from_experience(self, batch_size):\n",
    "        if self.memory_buffer.size() > batch_size:\n",
    "            idxs, states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)\n",
    "            td_loss = self.compute_td_loss(idxs, states, actions, rewards, next_states, dones)\n",
    "            self.optimizer.zero_grad()\n",
    "            td_loss.backward()\n",
    "            for param in self.DQN.parameters():\n",
    "                param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "            self.optimizer.step()\n",
    "            return(td_loss.item())\n",
    "        else:\n",
    "            return(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:     0, reward:   nan, loss: 0.000000, epsilon: 1.000000, episode:    0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:  1000, reward:   nan, loss: 0.000000, epsilon: 0.967544, episode:    0\n",
      "frames:  2000, reward: -20.000000, loss: 0.000000, epsilon: 0.936152, episode:    2\n",
      "frames:  3000, reward: -20.333333, loss: 0.000000, epsilon: 0.905789, episode:    3\n",
      "frames:  4000, reward: -20.500000, loss: 0.000000, epsilon: 0.876422, episode:    4\n",
      "frames:  5000, reward: -20.666667, loss: 0.000000, epsilon: 0.848017, episode:    6\n",
      "frames:  6000, reward: -20.714286, loss: 0.000000, epsilon: 0.820543, episode:    7\n",
      "frames:  7000, reward: -20.750000, loss: 0.000000, epsilon: 0.793971, episode:    8\n",
      "frames:  8000, reward: -20.777778, loss: 0.000000, epsilon: 0.768269, episode:    9\n",
      "frames:  9000, reward: -21.000000, loss: 0.000000, epsilon: 0.743410, episode:   11\n",
      "frames: 10000, reward: -21.000000, loss: 0.031564, epsilon: 0.719366, episode:   12\n",
      "frames: 11000, reward: -21.000000, loss: 0.055200, epsilon: 0.696110, episode:   13\n",
      "frames: 12000, reward: -21.000000, loss: 0.005563, epsilon: 0.673617, episode:   15\n",
      "frames: 13000, reward: -20.900000, loss: 0.033311, epsilon: 0.651861, episode:   16\n",
      "frames: 14000, reward: -20.900000, loss: 0.016033, epsilon: 0.630818, episode:   17\n",
      "frames: 15000, reward: -20.900000, loss: 0.042106, epsilon: 0.610465, episode:   18\n",
      "frames: 16000, reward: -20.900000, loss: 0.029667, epsilon: 0.590780, episode:   19\n",
      "frames: 17000, reward: -20.800000, loss: 0.061280, epsilon: 0.571740, episode:   20\n",
      "frames: 18000, reward: -20.800000, loss: 0.024004, epsilon: 0.553324, episode:   21\n",
      "frames: 19000, reward: -20.800000, loss: 0.019043, epsilon: 0.535511, episode:   22\n",
      "frames: 20000, reward: -20.700000, loss: 0.029863, epsilon: 0.518283, episode:   24\n",
      "frames: 21000, reward: -20.600000, loss: 0.021323, epsilon: 0.501619, episode:   25\n",
      "frames: 22000, reward: -20.700000, loss: 0.016894, epsilon: 0.485502, episode:   26\n",
      "frames: 23000, reward: -20.700000, loss: 0.004341, epsilon: 0.469913, episode:   27\n",
      "frames: 24000, reward: -20.700000, loss: 0.002743, epsilon: 0.454836, episode:   28\n",
      "frames: 25000, reward: -20.600000, loss: 0.005792, epsilon: 0.440252, episode:   29\n",
      "frames: 26000, reward: -20.700000, loss: 0.003874, epsilon: 0.426147, episode:   31\n",
      "frames: 27000, reward: -20.700000, loss: 0.005459, epsilon: 0.412504, episode:   32\n",
      "frames: 28000, reward: -20.800000, loss: 0.006686, epsilon: 0.399308, episode:   33\n",
      "frames: 29000, reward: -20.800000, loss: 0.002774, epsilon: 0.386545, episode:   34\n",
      "frames: 30000, reward: -20.900000, loss: 0.003141, epsilon: 0.374201, episode:   35\n",
      "frames: 31000, reward: -20.900000, loss: 0.005683, epsilon: 0.362261, episode:   37\n",
      "frames: 32000, reward: -20.900000, loss: 0.002378, epsilon: 0.350712, episode:   38\n",
      "frames: 33000, reward: -21.000000, loss: 0.003212, epsilon: 0.339542, episode:   39\n",
      "frames: 34000, reward: -20.900000, loss: 0.002583, epsilon: 0.328739, episode:   40\n",
      "frames: 35000, reward: -20.700000, loss: 0.002450, epsilon: 0.318289, episode:   41\n",
      "frames: 36000, reward: -20.700000, loss: 0.024395, epsilon: 0.308182, episode:   42\n",
      "frames: 37000, reward: -20.700000, loss: 0.004462, epsilon: 0.298407, episode:   43\n",
      "frames: 38000, reward: -20.600000, loss: 0.003588, epsilon: 0.288952, episode:   44\n",
      "frames: 39000, reward: -20.600000, loss: 0.013635, epsilon: 0.279806, episode:   45\n",
      "frames: 40000, reward: -20.400000, loss: 0.009783, epsilon: 0.270961, episode:   47\n",
      "frames: 41000, reward: -20.400000, loss: 0.008645, epsilon: 0.262406, episode:   47\n",
      "frames: 42000, reward: -20.100000, loss: 0.005560, epsilon: 0.254131, episode:   49\n",
      "frames: 43000, reward: -20.200000, loss: 0.009030, epsilon: 0.246127, episode:   50\n",
      "frames: 44000, reward: -20.300000, loss: 0.004096, epsilon: 0.238386, episode:   51\n",
      "frames: 45000, reward: -20.100000, loss: 0.003952, epsilon: 0.230899, episode:   52\n",
      "frames: 46000, reward: -20.100000, loss: 0.006707, epsilon: 0.223657, episode:   53\n",
      "frames: 47000, reward: -20.100000, loss: 0.006040, epsilon: 0.216652, episode:   54\n",
      "frames: 48000, reward: -19.900000, loss: 0.004411, epsilon: 0.209878, episode:   55\n",
      "frames: 49000, reward: -19.800000, loss: 0.006658, epsilon: 0.203325, episode:   56\n",
      "frames: 50000, reward: -19.900000, loss: 0.004876, epsilon: 0.196987, episode:   57\n",
      "frames: 51000, reward: -19.900000, loss: 0.004738, epsilon: 0.190857, episode:   58\n",
      "frames: 52000, reward: -19.900000, loss: 0.003407, epsilon: 0.184928, episode:   59\n",
      "frames: 53000, reward: -19.500000, loss: 0.005183, epsilon: 0.179193, episode:   60\n",
      "frames: 54000, reward: -19.500000, loss: 0.004067, epsilon: 0.173646, episode:   60\n",
      "frames: 55000, reward: -19.400000, loss: 0.002767, epsilon: 0.168281, episode:   61\n",
      "frames: 56000, reward: -19.600000, loss: 0.003998, epsilon: 0.163092, episode:   62\n",
      "frames: 57000, reward: -19.100000, loss: 0.006637, epsilon: 0.158073, episode:   63\n",
      "frames: 58000, reward: -19.200000, loss: 0.006189, epsilon: 0.153219, episode:   64\n",
      "frames: 59000, reward: -19.400000, loss: 0.010026, epsilon: 0.148523, episode:   65\n",
      "frames: 60000, reward: -19.500000, loss: 0.004405, epsilon: 0.143982, episode:   66\n",
      "frames: 61000, reward: -19.400000, loss: 0.007560, epsilon: 0.139589, episode:   67\n",
      "frames: 62000, reward: -19.400000, loss: 0.007116, epsilon: 0.135341, episode:   68\n",
      "frames: 63000, reward: -19.400000, loss: 0.014020, epsilon: 0.131232, episode:   68\n",
      "frames: 64000, reward: -19.300000, loss: 0.005005, epsilon: 0.127257, episode:   69\n",
      "frames: 65000, reward: -19.500000, loss: 0.004951, epsilon: 0.123413, episode:   70\n",
      "frames: 66000, reward: -19.500000, loss: 0.005251, epsilon: 0.119695, episode:   70\n",
      "frames: 67000, reward: -19.300000, loss: 0.004269, epsilon: 0.116099, episode:   71\n",
      "frames: 68000, reward: -19.300000, loss: 0.004026, epsilon: 0.112621, episode:   71\n",
      "frames: 69000, reward: -19.100000, loss: 0.009881, epsilon: 0.109256, episode:   72\n",
      "frames: 70000, reward: -18.800000, loss: 0.011710, epsilon: 0.106002, episode:   73\n",
      "frames: 71000, reward: -18.800000, loss: 0.008575, epsilon: 0.102855, episode:   73\n",
      "frames: 72000, reward: -18.300000, loss: 0.005609, epsilon: 0.099811, episode:   74\n",
      "frames: 73000, reward: -18.300000, loss: 0.004826, epsilon: 0.096866, episode:   74\n",
      "frames: 74000, reward: -18.000000, loss: 0.003586, epsilon: 0.094019, episode:   75\n",
      "frames: 75000, reward: -17.700000, loss: 0.006667, epsilon: 0.091264, episode:   76\n",
      "frames: 76000, reward: -17.700000, loss: 0.005648, epsilon: 0.088600, episode:   76\n",
      "frames: 77000, reward: -17.700000, loss: 0.004202, epsilon: 0.086023, episode:   77\n",
      "frames: 78000, reward: -17.700000, loss: 0.013091, epsilon: 0.083531, episode:   78\n",
      "frames: 79000, reward: -17.700000, loss: 0.004990, epsilon: 0.081120, episode:   78\n",
      "frames: 80000, reward: -17.600000, loss: 0.008153, epsilon: 0.078789, episode:   79\n",
      "frames: 81000, reward: -17.600000, loss: 0.005483, epsilon: 0.076533, episode:   79\n",
      "frames: 82000, reward: -17.600000, loss: 0.005808, epsilon: 0.074352, episode:   80\n",
      "frames: 83000, reward: -17.600000, loss: 0.001644, epsilon: 0.072243, episode:   80\n",
      "frames: 84000, reward: -17.700000, loss: 0.003764, epsilon: 0.070202, episode:   81\n",
      "frames: 85000, reward: -17.700000, loss: 0.005031, epsilon: 0.068228, episode:   81\n",
      "frames: 86000, reward: -17.500000, loss: 0.003535, epsilon: 0.066319, episode:   82\n",
      "frames: 87000, reward: -17.500000, loss: 0.005015, epsilon: 0.064473, episode:   82\n",
      "frames: 88000, reward: -17.600000, loss: 0.004492, epsilon: 0.062687, episode:   83\n",
      "frames: 89000, reward: -17.600000, loss: 0.002477, epsilon: 0.060960, episode:   83\n",
      "frames: 90000, reward: -17.600000, loss: 0.005089, epsilon: 0.059289, episode:   83\n",
      "frames: 91000, reward: -17.100000, loss: 0.001446, epsilon: 0.057673, episode:   84\n",
      "frames: 92000, reward: -17.100000, loss: 0.003673, epsilon: 0.056110, episode:   84\n",
      "frames: 93000, reward: -17.200000, loss: 0.001830, epsilon: 0.054599, episode:   85\n",
      "frames: 94000, reward: -17.100000, loss: 0.003149, epsilon: 0.053137, episode:   86\n",
      "frames: 95000, reward: -17.100000, loss: 0.001694, epsilon: 0.051722, episode:   86\n",
      "frames: 96000, reward: -16.900000, loss: 0.004203, epsilon: 0.050355, episode:   87\n",
      "frames: 97000, reward: -16.900000, loss: 0.002999, epsilon: 0.049032, episode:   87\n",
      "frames: 98000, reward: -16.900000, loss: 0.002230, epsilon: 0.047752, episode:   88\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 99000, reward: -16.900000, loss: 0.003382, epsilon: 0.046514, episode:   88\n",
      "frames: 100000, reward: -16.900000, loss: 0.002754, epsilon: 0.045317, episode:   88\n",
      "frames: 101000, reward: -16.000000, loss: 0.003562, epsilon: 0.044159, episode:   89\n",
      "frames: 102000, reward: -16.000000, loss: 0.006249, epsilon: 0.043040, episode:   89\n",
      "frames: 103000, reward: -13.300000, loss: 0.001905, epsilon: 0.041956, episode:   90\n",
      "frames: 104000, reward: -13.300000, loss: 0.003652, epsilon: 0.040909, episode:   90\n",
      "frames: 105000, reward: -13.300000, loss: 0.003893, epsilon: 0.039895, episode:   90\n",
      "frames: 106000, reward: -13.300000, loss: 0.002346, epsilon: 0.038915, episode:   90\n",
      "frames: 107000, reward: -12.100000, loss: 0.002709, epsilon: 0.037967, episode:   91\n",
      "frames: 108000, reward: -12.100000, loss: 0.002131, epsilon: 0.037050, episode:   91\n",
      "frames: 109000, reward: -12.100000, loss: 0.002447, epsilon: 0.036164, episode:   91\n",
      "frames: 110000, reward: -12.000000, loss: 0.002022, epsilon: 0.035306, episode:   92\n",
      "frames: 111000, reward: -12.000000, loss: 0.003518, epsilon: 0.034476, episode:   92\n",
      "frames: 112000, reward: -12.100000, loss: 0.002449, epsilon: 0.033674, episode:   93\n",
      "frames: 113000, reward: -12.100000, loss: 0.002626, epsilon: 0.032898, episode:   93\n",
      "frames: 114000, reward: -12.100000, loss: 0.003248, epsilon: 0.032147, episode:   93\n",
      "frames: 115000, reward: -12.700000, loss: 0.001866, epsilon: 0.031421, episode:   94\n",
      "frames: 116000, reward: -12.700000, loss: 0.001908, epsilon: 0.030719, episode:   94\n",
      "frames: 117000, reward: -12.700000, loss: 0.001299, epsilon: 0.030039, episode:   94\n",
      "frames: 118000, reward: -12.700000, loss: 0.002071, epsilon: 0.029383, episode:   94\n",
      "frames: 119000, reward: -10.700000, loss: 0.002659, epsilon: 0.028747, episode:   95\n",
      "frames: 120000, reward: -10.700000, loss: 0.001987, epsilon: 0.028132, episode:   95\n",
      "frames: 121000, reward: -10.700000, loss: 0.002227, epsilon: 0.027538, episode:   95\n",
      "frames: 122000, reward: -9.200000, loss: 0.001392, epsilon: 0.026963, episode:   96\n",
      "frames: 123000, reward: -9.200000, loss: 0.001816, epsilon: 0.026407, episode:   96\n",
      "frames: 124000, reward: -9.100000, loss: 0.002843, epsilon: 0.025869, episode:   97\n",
      "frames: 125000, reward: -9.100000, loss: 0.002075, epsilon: 0.025349, episode:   97\n",
      "frames: 126000, reward: -9.100000, loss: 0.001581, epsilon: 0.024846, episode:   97\n",
      "frames: 127000, reward: -8.300000, loss: 0.002420, epsilon: 0.024359, episode:   98\n",
      "frames: 128000, reward: -8.300000, loss: 0.003081, epsilon: 0.023888, episode:   98\n",
      "frames: 129000, reward: -8.300000, loss: 0.001393, epsilon: 0.023433, episode:   98\n",
      "frames: 130000, reward: -6.800000, loss: 0.001729, epsilon: 0.022992, episode:   99\n",
      "frames: 131000, reward: -6.800000, loss: 0.001805, epsilon: 0.022567, episode:   99\n",
      "frames: 132000, reward: -6.800000, loss: 0.002667, epsilon: 0.022155, episode:   99\n",
      "frames: 133000, reward: -8.500000, loss: 0.004024, epsilon: 0.021756, episode:  100\n",
      "frames: 134000, reward: -8.500000, loss: 0.001310, epsilon: 0.021371, episode:  100\n",
      "frames: 135000, reward: -9.200000, loss: 0.001763, epsilon: 0.020998, episode:  101\n",
      "frames: 136000, reward: -9.200000, loss: 0.001955, epsilon: 0.020637, episode:  101\n",
      "frames: 137000, reward: -8.600000, loss: 0.003319, epsilon: 0.020289, episode:  102\n",
      "frames: 138000, reward: -8.600000, loss: 0.001739, epsilon: 0.019951, episode:  102\n",
      "frames: 139000, reward: -8.600000, loss: 0.002790, epsilon: 0.019625, episode:  102\n",
      "frames: 140000, reward: -8.200000, loss: 0.001374, epsilon: 0.019310, episode:  103\n",
      "frames: 141000, reward: -8.200000, loss: 0.002266, epsilon: 0.019004, episode:  103\n",
      "frames: 142000, reward: -7.600000, loss: 0.003796, epsilon: 0.018709, episode:  104\n",
      "frames: 143000, reward: -7.600000, loss: 0.005230, epsilon: 0.018424, episode:  104\n",
      "frames: 144000, reward: -7.600000, loss: 0.004248, epsilon: 0.018147, episode:  104\n",
      "frames: 145000, reward: -9.100000, loss: 0.003205, epsilon: 0.017880, episode:  105\n",
      "frames: 146000, reward: -9.100000, loss: 0.001378, epsilon: 0.017622, episode:  105\n",
      "frames: 147000, reward: -10.000000, loss: 0.001982, epsilon: 0.017372, episode:  106\n",
      "frames: 148000, reward: -10.000000, loss: 0.002156, epsilon: 0.017130, episode:  106\n",
      "frames: 149000, reward: -10.000000, loss: 0.002141, epsilon: 0.016897, episode:  106\n",
      "frames: 150000, reward: -10.000000, loss: 0.003349, epsilon: 0.016671, episode:  106\n",
      "frames: 151000, reward: -8.600000, loss: 0.002073, epsilon: 0.016452, episode:  107\n",
      "frames: 152000, reward: -8.600000, loss: 0.002473, epsilon: 0.016240, episode:  107\n",
      "frames: 153000, reward: -8.500000, loss: 0.003079, epsilon: 0.016036, episode:  108\n",
      "frames: 154000, reward: -8.500000, loss: 0.002808, epsilon: 0.015838, episode:  108\n",
      "frames: 155000, reward: -8.500000, loss: 0.002625, epsilon: 0.015647, episode:  108\n",
      "frames: 156000, reward: -8.500000, loss: 0.003458, epsilon: 0.015461, episode:  108\n",
      "frames: 157000, reward: -9.600000, loss: 0.002839, epsilon: 0.015282, episode:  109\n",
      "frames: 158000, reward: -9.600000, loss: 0.002970, epsilon: 0.015109, episode:  109\n",
      "frames: 159000, reward: -9.600000, loss: 0.001965, epsilon: 0.014942, episode:  109\n",
      "frames: 160000, reward: -9.200000, loss: 0.002924, epsilon: 0.014780, episode:  110\n",
      "frames: 161000, reward: -9.200000, loss: 0.002756, epsilon: 0.014623, episode:  110\n",
      "frames: 162000, reward: -9.200000, loss: 0.002457, epsilon: 0.014471, episode:  110\n",
      "frames: 163000, reward: -8.400000, loss: 0.001457, epsilon: 0.014325, episode:  111\n",
      "frames: 164000, reward: -8.400000, loss: 0.001761, epsilon: 0.014183, episode:  111\n",
      "frames: 165000, reward: -8.400000, loss: 0.001318, epsilon: 0.014046, episode:  111\n",
      "frames: 166000, reward: -8.400000, loss: 0.001925, epsilon: 0.013913, episode:  111\n",
      "frames: 167000, reward: -6.700000, loss: 0.001215, epsilon: 0.013785, episode:  112\n",
      "frames: 168000, reward: -6.700000, loss: 0.001316, epsilon: 0.013661, episode:  112\n",
      "frames: 169000, reward: -6.700000, loss: 0.001944, epsilon: 0.013541, episode:  112\n",
      "frames: 170000, reward: -6.400000, loss: 0.004168, epsilon: 0.013425, episode:  113\n",
      "frames: 171000, reward: -6.400000, loss: 0.001929, epsilon: 0.013313, episode:  113\n",
      "frames: 172000, reward: -6.400000, loss: 0.002224, epsilon: 0.013204, episode:  113\n",
      "frames: 173000, reward: -6.200000, loss: 0.001906, epsilon: 0.013099, episode:  114\n",
      "frames: 174000, reward: -6.200000, loss: 0.003835, epsilon: 0.012997, episode:  114\n",
      "frames: 175000, reward: -6.200000, loss: 0.003072, epsilon: 0.012899, episode:  114\n",
      "frames: 176000, reward: -4.300000, loss: 0.001628, epsilon: 0.012804, episode:  115\n",
      "frames: 177000, reward: -4.300000, loss: 0.002748, epsilon: 0.012712, episode:  115\n",
      "frames: 178000, reward: -4.300000, loss: 0.002056, epsilon: 0.012623, episode:  115\n",
      "frames: 179000, reward: -4.300000, loss: 0.002332, epsilon: 0.012537, episode:  115\n",
      "frames: 180000, reward: -3.200000, loss: 0.002532, epsilon: 0.012454, episode:  116\n",
      "frames: 181000, reward: -3.200000, loss: 0.001262, epsilon: 0.012374, episode:  116\n",
      "frames: 182000, reward: -3.200000, loss: 0.008254, epsilon: 0.012296, episode:  116\n",
      "frames: 183000, reward: -3.900000, loss: 0.001168, epsilon: 0.012220, episode:  117\n",
      "frames: 184000, reward: -3.900000, loss: 0.001597, epsilon: 0.012148, episode:  117\n",
      "frames: 185000, reward: -3.900000, loss: 0.002843, epsilon: 0.012077, episode:  117\n",
      "frames: 186000, reward: -3.900000, loss: 0.002340, epsilon: 0.012009, episode:  117\n",
      "frames: 187000, reward: -3.200000, loss: 0.001768, epsilon: 0.011943, episode:  118\n",
      "frames: 188000, reward: -3.200000, loss: 0.002894, epsilon: 0.011880, episode:  118\n",
      "frames: 189000, reward: -3.200000, loss: 0.003005, epsilon: 0.011818, episode:  118\n",
      "frames: 190000, reward: -3.200000, loss: 0.002843, epsilon: 0.011758, episode:  118\n",
      "frames: 191000, reward: -3.100000, loss: 0.001535, epsilon: 0.011701, episode:  119\n",
      "frames: 192000, reward: -3.100000, loss: 0.001811, epsilon: 0.011645, episode:  119\n",
      "frames: 193000, reward: -3.100000, loss: 0.001776, epsilon: 0.011591, episode:  119\n",
      "frames: 194000, reward: -2.900000, loss: 0.002270, epsilon: 0.011539, episode:  120\n",
      "frames: 195000, reward: -2.900000, loss: 0.001116, epsilon: 0.011488, episode:  120\n",
      "frames: 196000, reward: -2.900000, loss: 0.001926, epsilon: 0.011440, episode:  120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 197000, reward: -3.500000, loss: 0.002203, epsilon: 0.011392, episode:  121\n",
      "frames: 198000, reward: -3.500000, loss: 0.002144, epsilon: 0.011347, episode:  121\n",
      "frames: 199000, reward: -3.500000, loss: 0.000842, epsilon: 0.011303, episode:  121\n",
      "frames: 200000, reward: -3.500000, loss: 0.002038, epsilon: 0.011260, episode:  121\n",
      "frames: 201000, reward: -5.000000, loss: 0.001565, epsilon: 0.011219, episode:  122\n",
      "frames: 202000, reward: -5.000000, loss: 0.002008, epsilon: 0.011179, episode:  122\n",
      "frames: 203000, reward: -5.000000, loss: 0.001313, epsilon: 0.011140, episode:  122\n",
      "frames: 204000, reward: -5.000000, loss: 0.001217, epsilon: 0.011103, episode:  122\n",
      "frames: 205000, reward: -4.000000, loss: 0.001761, epsilon: 0.011066, episode:  123\n",
      "frames: 206000, reward: -4.000000, loss: 0.001378, epsilon: 0.011032, episode:  123\n",
      "frames: 207000, reward: -4.700000, loss: 0.001267, epsilon: 0.010998, episode:  124\n",
      "frames: 208000, reward: -4.700000, loss: 0.002051, epsilon: 0.010965, episode:  124\n",
      "frames: 209000, reward: -4.700000, loss: 0.002657, epsilon: 0.010933, episode:  124\n",
      "frames: 210000, reward: -4.700000, loss: 0.001196, epsilon: 0.010903, episode:  124\n",
      "frames: 211000, reward: -5.900000, loss: 0.001098, epsilon: 0.010873, episode:  125\n",
      "frames: 212000, reward: -5.900000, loss: 0.001510, epsilon: 0.010845, episode:  125\n",
      "frames: 213000, reward: -5.900000, loss: 0.002543, epsilon: 0.010817, episode:  125\n",
      "frames: 214000, reward: -5.900000, loss: 0.001762, epsilon: 0.010790, episode:  125\n",
      "frames: 215000, reward: -5.900000, loss: 0.001409, epsilon: 0.010764, episode:  125\n",
      "frames: 216000, reward: -6.600000, loss: 0.001080, epsilon: 0.010739, episode:  126\n",
      "frames: 217000, reward: -6.600000, loss: 0.001533, epsilon: 0.010715, episode:  126\n",
      "frames: 218000, reward: -6.600000, loss: 0.001984, epsilon: 0.010691, episode:  126\n",
      "frames: 219000, reward: -6.600000, loss: 0.001313, epsilon: 0.010669, episode:  126\n",
      "frames: 220000, reward: -5.400000, loss: 0.001123, epsilon: 0.010647, episode:  127\n",
      "frames: 221000, reward: -5.400000, loss: 0.002061, epsilon: 0.010626, episode:  127\n",
      "frames: 222000, reward: -5.400000, loss: 0.001655, epsilon: 0.010605, episode:  127\n",
      "frames: 223000, reward: -5.400000, loss: 0.001733, epsilon: 0.010585, episode:  127\n",
      "frames: 224000, reward: -6.400000, loss: 0.001556, epsilon: 0.010566, episode:  128\n",
      "frames: 225000, reward: -6.400000, loss: 0.001906, epsilon: 0.010548, episode:  128\n",
      "frames: 226000, reward: -6.400000, loss: 0.002196, epsilon: 0.010530, episode:  128\n",
      "frames: 227000, reward: -6.400000, loss: 0.001452, epsilon: 0.010512, episode:  128\n",
      "frames: 228000, reward: -6.600000, loss: 0.008017, epsilon: 0.010495, episode:  129\n",
      "frames: 229000, reward: -6.600000, loss: 0.001058, epsilon: 0.010479, episode:  129\n",
      "frames: 230000, reward: -6.600000, loss: 0.000897, epsilon: 0.010463, episode:  129\n",
      "frames: 231000, reward: -6.600000, loss: 0.001097, epsilon: 0.010448, episode:  129\n",
      "frames: 232000, reward: -6.600000, loss: 0.001655, epsilon: 0.010434, episode:  129\n",
      "frames: 233000, reward: -6.400000, loss: 0.001642, epsilon: 0.010419, episode:  130\n",
      "frames: 234000, reward: -6.400000, loss: 0.001622, epsilon: 0.010406, episode:  130\n",
      "frames: 235000, reward: -6.400000, loss: 0.001025, epsilon: 0.010392, episode:  130\n",
      "frames: 236000, reward: -6.400000, loss: 0.001360, epsilon: 0.010379, episode:  130\n",
      "frames: 237000, reward: -5.600000, loss: 0.001013, epsilon: 0.010367, episode:  131\n",
      "frames: 238000, reward: -5.600000, loss: 0.001308, epsilon: 0.010355, episode:  131\n",
      "frames: 239000, reward: -5.600000, loss: 0.003187, epsilon: 0.010343, episode:  131\n",
      "frames: 240000, reward: -5.100000, loss: 0.002409, epsilon: 0.010332, episode:  132\n",
      "frames: 241000, reward: -5.100000, loss: 0.001403, epsilon: 0.010321, episode:  132\n",
      "frames: 242000, reward: -5.100000, loss: 0.000934, epsilon: 0.010311, episode:  132\n",
      "frames: 243000, reward: -5.100000, loss: 0.001287, epsilon: 0.010301, episode:  132\n",
      "frames: 244000, reward: -5.100000, loss: 0.004092, epsilon: 0.010291, episode:  132\n",
      "frames: 245000, reward: -4.600000, loss: 0.002566, epsilon: 0.010281, episode:  133\n",
      "frames: 246000, reward: -4.600000, loss: 0.002986, epsilon: 0.010272, episode:  133\n",
      "frames: 247000, reward: -4.600000, loss: 0.001372, epsilon: 0.010263, episode:  133\n",
      "frames: 248000, reward: -4.600000, loss: 0.001023, epsilon: 0.010254, episode:  133\n",
      "frames: 249000, reward: -2.100000, loss: 0.001647, epsilon: 0.010246, episode:  134\n",
      "frames: 250000, reward: -2.100000, loss: 0.001474, epsilon: 0.010238, episode:  134\n",
      "frames: 251000, reward: -2.100000, loss: 0.001450, epsilon: 0.010230, episode:  134\n",
      "frames: 252000, reward: -2.100000, loss: 0.000810, epsilon: 0.010223, episode:  134\n",
      "frames: 253000, reward: -1.100000, loss: 0.000868, epsilon: 0.010215, episode:  135\n",
      "frames: 254000, reward: -1.100000, loss: 0.001124, epsilon: 0.010208, episode:  135\n",
      "frames: 255000, reward: -1.100000, loss: 0.002588, epsilon: 0.010201, episode:  135\n",
      "frames: 256000, reward: -1.100000, loss: 0.001428, epsilon: 0.010195, episode:  135\n",
      "frames: 257000, reward: -0.900000, loss: 0.000943, epsilon: 0.010188, episode:  136\n",
      "frames: 258000, reward: -0.900000, loss: 0.001168, epsilon: 0.010182, episode:  136\n",
      "frames: 259000, reward: -0.900000, loss: 0.001221, epsilon: 0.010176, episode:  136\n",
      "frames: 260000, reward: -0.900000, loss: 0.001914, epsilon: 0.010171, episode:  136\n",
      "frames: 261000, reward: -0.500000, loss: 0.001521, epsilon: 0.010165, episode:  137\n",
      "frames: 262000, reward: -0.500000, loss: 0.000645, epsilon: 0.010160, episode:  137\n",
      "frames: 263000, reward: -0.500000, loss: 0.000988, epsilon: 0.010154, episode:  137\n",
      "frames: 264000, reward: 1.600000, loss: 0.001070, epsilon: 0.010149, episode:  138\n",
      "frames: 265000, reward: 1.600000, loss: 0.001517, epsilon: 0.010144, episode:  138\n",
      "frames: 266000, reward: 1.600000, loss: 0.001257, epsilon: 0.010140, episode:  138\n",
      "frames: 267000, reward: 1.600000, loss: 0.001341, epsilon: 0.010135, episode:  138\n",
      "frames: 268000, reward: 2.000000, loss: 0.001637, epsilon: 0.010131, episode:  139\n",
      "frames: 269000, reward: 2.000000, loss: 0.001615, epsilon: 0.010126, episode:  139\n",
      "frames: 270000, reward: 2.000000, loss: 0.002676, epsilon: 0.010122, episode:  139\n",
      "frames: 271000, reward: 2.000000, loss: 0.001486, epsilon: 0.010118, episode:  139\n",
      "frames: 272000, reward: 2.600000, loss: 0.000922, epsilon: 0.010114, episode:  140\n",
      "frames: 273000, reward: 2.600000, loss: 0.001242, epsilon: 0.010111, episode:  140\n",
      "frames: 274000, reward: 2.600000, loss: 0.001508, epsilon: 0.010107, episode:  140\n",
      "frames: 275000, reward: 3.600000, loss: 0.001310, epsilon: 0.010103, episode:  141\n",
      "frames: 276000, reward: 3.600000, loss: 0.002686, epsilon: 0.010100, episode:  141\n",
      "frames: 277000, reward: 3.600000, loss: 0.001775, epsilon: 0.010097, episode:  141\n",
      "frames: 278000, reward: 5.000000, loss: 0.002977, epsilon: 0.010094, episode:  142\n",
      "frames: 279000, reward: 5.000000, loss: 0.001937, epsilon: 0.010091, episode:  142\n",
      "frames: 280000, reward: 5.000000, loss: 0.001414, epsilon: 0.010088, episode:  142\n",
      "frames: 281000, reward: 5.000000, loss: 0.001258, epsilon: 0.010085, episode:  142\n",
      "frames: 282000, reward: 4.900000, loss: 0.000665, epsilon: 0.010082, episode:  143\n",
      "frames: 283000, reward: 4.900000, loss: 0.001140, epsilon: 0.010079, episode:  143\n",
      "frames: 284000, reward: 4.900000, loss: 0.000720, epsilon: 0.010077, episode:  143\n",
      "frames: 285000, reward: 5.800000, loss: 0.001731, epsilon: 0.010074, episode:  144\n",
      "frames: 286000, reward: 5.800000, loss: 0.002143, epsilon: 0.010072, episode:  144\n",
      "frames: 287000, reward: 5.800000, loss: 0.001266, epsilon: 0.010069, episode:  144\n",
      "frames: 288000, reward: 5.800000, loss: 0.001190, epsilon: 0.010067, episode:  144\n",
      "frames: 289000, reward: 5.100000, loss: 0.001368, epsilon: 0.010065, episode:  145\n",
      "frames: 290000, reward: 5.100000, loss: 0.001450, epsilon: 0.010063, episode:  145\n",
      "frames: 291000, reward: 7.400000, loss: 0.001897, epsilon: 0.010061, episode:  146\n",
      "frames: 292000, reward: 7.400000, loss: 0.001070, epsilon: 0.010059, episode:  146\n",
      "frames: 293000, reward: 7.400000, loss: 0.001320, epsilon: 0.010057, episode:  146\n",
      "frames: 294000, reward: 7.500000, loss: 0.002348, epsilon: 0.010055, episode:  147\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 295000, reward: 7.500000, loss: 0.001837, epsilon: 0.010053, episode:  147\n",
      "frames: 296000, reward: 8.600000, loss: 0.002585, epsilon: 0.010051, episode:  148\n",
      "frames: 297000, reward: 8.600000, loss: 0.001031, epsilon: 0.010050, episode:  148\n",
      "frames: 298000, reward: 8.600000, loss: 0.000800, epsilon: 0.010048, episode:  148\n",
      "frames: 299000, reward: 8.600000, loss: 0.000783, epsilon: 0.010046, episode:  148\n",
      "frames: 300000, reward: 8.600000, loss: 0.001234, epsilon: 0.010045, episode:  148\n",
      "frames: 301000, reward: 9.200000, loss: 0.000966, epsilon: 0.010043, episode:  149\n",
      "frames: 302000, reward: 9.200000, loss: 0.001301, epsilon: 0.010042, episode:  149\n",
      "frames: 303000, reward: 10.400000, loss: 0.001291, epsilon: 0.010041, episode:  150\n",
      "frames: 304000, reward: 10.400000, loss: 0.001706, epsilon: 0.010039, episode:  150\n",
      "frames: 305000, reward: 10.400000, loss: 0.001655, epsilon: 0.010038, episode:  150\n",
      "frames: 306000, reward: 10.400000, loss: 0.000862, epsilon: 0.010037, episode:  150\n",
      "frames: 307000, reward: 10.000000, loss: 0.001165, epsilon: 0.010036, episode:  151\n",
      "frames: 308000, reward: 10.000000, loss: 0.001136, epsilon: 0.010034, episode:  151\n",
      "frames: 309000, reward: 10.800000, loss: 0.001407, epsilon: 0.010033, episode:  152\n",
      "frames: 310000, reward: 10.800000, loss: 0.008842, epsilon: 0.010032, episode:  152\n",
      "frames: 311000, reward: 10.800000, loss: 0.001102, epsilon: 0.010031, episode:  152\n",
      "frames: 312000, reward: 11.800000, loss: 0.000999, epsilon: 0.010030, episode:  153\n",
      "frames: 313000, reward: 11.800000, loss: 0.001141, epsilon: 0.010029, episode:  153\n",
      "frames: 314000, reward: 11.900000, loss: 0.001642, epsilon: 0.010028, episode:  154\n",
      "frames: 315000, reward: 11.900000, loss: 0.001170, epsilon: 0.010027, episode:  154\n",
      "frames: 316000, reward: 11.900000, loss: 0.007074, epsilon: 0.010026, episode:  154\n",
      "frames: 317000, reward: 11.600000, loss: 0.001389, epsilon: 0.010026, episode:  155\n",
      "frames: 318000, reward: 11.600000, loss: 0.000817, epsilon: 0.010025, episode:  155\n",
      "frames: 319000, reward: 11.800000, loss: 0.000935, epsilon: 0.010024, episode:  156\n",
      "frames: 320000, reward: 11.800000, loss: 0.000517, epsilon: 0.010023, episode:  156\n",
      "frames: 321000, reward: 13.200000, loss: 0.000705, epsilon: 0.010022, episode:  157\n",
      "frames: 322000, reward: 13.200000, loss: 0.005386, epsilon: 0.010022, episode:  157\n",
      "frames: 323000, reward: 12.800000, loss: 0.001202, epsilon: 0.010021, episode:  158\n",
      "frames: 324000, reward: 12.800000, loss: 0.001372, epsilon: 0.010020, episode:  158\n",
      "frames: 325000, reward: 14.100000, loss: 0.001129, epsilon: 0.010020, episode:  159\n",
      "frames: 326000, reward: 14.100000, loss: 0.000744, epsilon: 0.010019, episode:  159\n",
      "frames: 327000, reward: 14.300000, loss: 0.001153, epsilon: 0.010018, episode:  160\n",
      "frames: 328000, reward: 14.300000, loss: 0.000832, epsilon: 0.010018, episode:  160\n",
      "frames: 329000, reward: 14.300000, loss: 0.001577, epsilon: 0.010017, episode:  160\n",
      "frames: 330000, reward: 14.300000, loss: 0.000895, epsilon: 0.010017, episode:  160\n",
      "frames: 331000, reward: 14.200000, loss: 0.001561, epsilon: 0.010016, episode:  161\n",
      "frames: 332000, reward: 14.200000, loss: 0.001408, epsilon: 0.010015, episode:  161\n",
      "frames: 333000, reward: 14.200000, loss: 0.001138, epsilon: 0.010015, episode:  161\n",
      "frames: 334000, reward: 12.600000, loss: 0.001975, epsilon: 0.010014, episode:  162\n",
      "frames: 335000, reward: 12.600000, loss: 0.000601, epsilon: 0.010014, episode:  162\n",
      "frames: 336000, reward: 13.000000, loss: 0.001603, epsilon: 0.010014, episode:  163\n",
      "frames: 337000, reward: 13.000000, loss: 0.000699, epsilon: 0.010013, episode:  163\n",
      "frames: 338000, reward: 13.100000, loss: 0.000562, epsilon: 0.010013, episode:  164\n",
      "frames: 339000, reward: 15.800000, loss: 0.002589, epsilon: 0.010012, episode:  165\n",
      "frames: 340000, reward: 15.800000, loss: 0.001565, epsilon: 0.010012, episode:  165\n",
      "frames: 341000, reward: 15.800000, loss: 0.000642, epsilon: 0.010011, episode:  165\n",
      "frames: 342000, reward: 15.800000, loss: 0.000714, epsilon: 0.010011, episode:  165\n",
      "frames: 343000, reward: 13.800000, loss: 0.009995, epsilon: 0.010011, episode:  166\n",
      "frames: 344000, reward: 13.800000, loss: 0.002627, epsilon: 0.010010, episode:  166\n",
      "frames: 345000, reward: 13.800000, loss: 0.001093, epsilon: 0.010010, episode:  166\n",
      "frames: 346000, reward: 13.800000, loss: 0.001401, epsilon: 0.010010, episode:  166\n",
      "frames: 347000, reward: 12.700000, loss: 0.002003, epsilon: 0.010009, episode:  167\n",
      "frames: 348000, reward: 12.700000, loss: 0.003003, epsilon: 0.010009, episode:  167\n",
      "frames: 349000, reward: 11.900000, loss: 0.003417, epsilon: 0.010009, episode:  168\n",
      "frames: 350000, reward: 11.900000, loss: 0.001329, epsilon: 0.010008, episode:  168\n",
      "frames: 351000, reward: 11.900000, loss: 0.001508, epsilon: 0.010008, episode:  168\n",
      "frames: 352000, reward: 10.900000, loss: 0.001286, epsilon: 0.010008, episode:  169\n",
      "frames: 353000, reward: 10.900000, loss: 0.001363, epsilon: 0.010008, episode:  169\n",
      "frames: 354000, reward: 10.900000, loss: 0.000911, epsilon: 0.010007, episode:  170\n",
      "frames: 355000, reward: 10.900000, loss: 0.001567, epsilon: 0.010007, episode:  170\n",
      "frames: 356000, reward: 10.900000, loss: 0.001360, epsilon: 0.010007, episode:  170\n",
      "frames: 357000, reward: 12.000000, loss: 0.001527, epsilon: 0.010007, episode:  171\n",
      "frames: 358000, reward: 12.000000, loss: 0.001883, epsilon: 0.010007, episode:  171\n",
      "frames: 359000, reward: 12.000000, loss: 0.001742, epsilon: 0.010006, episode:  171\n",
      "frames: 360000, reward: 12.600000, loss: 0.001549, epsilon: 0.010006, episode:  172\n",
      "frames: 361000, reward: 12.600000, loss: 0.001159, epsilon: 0.010006, episode:  172\n",
      "frames: 362000, reward: 12.500000, loss: 0.001161, epsilon: 0.010006, episode:  173\n",
      "frames: 363000, reward: 12.500000, loss: 0.001999, epsilon: 0.010006, episode:  173\n",
      "frames: 364000, reward: 12.500000, loss: 0.001078, epsilon: 0.010005, episode:  173\n",
      "frames: 365000, reward: 11.700000, loss: 0.000838, epsilon: 0.010005, episode:  174\n",
      "frames: 366000, reward: 11.700000, loss: 0.000733, epsilon: 0.010005, episode:  174\n",
      "frames: 367000, reward: 11.500000, loss: 0.001006, epsilon: 0.010005, episode:  175\n",
      "frames: 368000, reward: 13.500000, loss: 0.000623, epsilon: 0.010005, episode:  176\n",
      "frames: 369000, reward: 13.500000, loss: 0.000949, epsilon: 0.010005, episode:  176\n",
      "frames: 370000, reward: 14.200000, loss: 0.000573, epsilon: 0.010004, episode:  177\n",
      "frames: 371000, reward: 14.200000, loss: 0.000678, epsilon: 0.010004, episode:  177\n",
      "frames: 372000, reward: 15.300000, loss: 0.000676, epsilon: 0.010004, episode:  178\n",
      "frames: 373000, reward: 15.300000, loss: 0.000668, epsilon: 0.010004, episode:  178\n",
      "frames: 374000, reward: 16.500000, loss: 0.000496, epsilon: 0.010004, episode:  179\n",
      "frames: 375000, reward: 16.500000, loss: 0.000578, epsilon: 0.010004, episode:  179\n",
      "frames: 376000, reward: 13.100000, loss: 0.001353, epsilon: 0.010004, episode:  180\n",
      "frames: 377000, reward: 13.100000, loss: 0.000436, epsilon: 0.010003, episode:  180\n",
      "frames: 378000, reward: 13.600000, loss: 0.001102, epsilon: 0.010003, episode:  181\n",
      "frames: 379000, reward: 13.600000, loss: 0.000403, epsilon: 0.010003, episode:  181\n",
      "frames: 380000, reward: 14.800000, loss: 0.000516, epsilon: 0.010003, episode:  182\n",
      "frames: 381000, reward: 14.800000, loss: 0.000719, epsilon: 0.010003, episode:  182\n",
      "frames: 382000, reward: 14.700000, loss: 0.000743, epsilon: 0.010003, episode:  183\n",
      "frames: 383000, reward: 14.700000, loss: 0.000862, epsilon: 0.010003, episode:  183\n",
      "frames: 384000, reward: 15.300000, loss: 0.000621, epsilon: 0.010003, episode:  184\n",
      "frames: 385000, reward: 15.500000, loss: 0.001098, epsilon: 0.010003, episode:  185\n",
      "frames: 386000, reward: 15.500000, loss: 0.000414, epsilon: 0.010003, episode:  185\n",
      "frames: 387000, reward: 15.000000, loss: 0.001658, epsilon: 0.010002, episode:  186\n",
      "frames: 388000, reward: 15.000000, loss: 0.000713, epsilon: 0.010002, episode:  186\n",
      "frames: 389000, reward: 15.000000, loss: 0.001394, epsilon: 0.010002, episode:  186\n",
      "frames: 390000, reward: 14.800000, loss: 0.001472, epsilon: 0.010002, episode:  187\n",
      "frames: 391000, reward: 14.800000, loss: 0.000944, epsilon: 0.010002, episode:  187\n",
      "frames: 392000, reward: 14.700000, loss: 0.000765, epsilon: 0.010002, episode:  188\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 393000, reward: 14.900000, loss: 0.001302, epsilon: 0.010002, episode:  189\n",
      "frames: 394000, reward: 14.900000, loss: 0.000856, epsilon: 0.010002, episode:  189\n",
      "frames: 395000, reward: 14.900000, loss: 0.000952, epsilon: 0.010002, episode:  189\n",
      "frames: 396000, reward: 17.900000, loss: 0.000888, epsilon: 0.010002, episode:  190\n",
      "frames: 397000, reward: 17.900000, loss: 0.000677, epsilon: 0.010002, episode:  190\n",
      "frames: 398000, reward: 18.000000, loss: 0.001133, epsilon: 0.010002, episode:  191\n",
      "frames: 399000, reward: 18.000000, loss: 0.001418, epsilon: 0.010002, episode:  191\n",
      "frames: 400000, reward: 18.000000, loss: 0.000889, epsilon: 0.010002, episode:  191\n",
      "frames: 401000, reward: 16.800000, loss: 0.000782, epsilon: 0.010002, episode:  192\n",
      "frames: 402000, reward: 17.000000, loss: 0.000673, epsilon: 0.010001, episode:  193\n",
      "frames: 403000, reward: 17.000000, loss: 0.000903, epsilon: 0.010001, episode:  193\n",
      "frames: 404000, reward: 17.000000, loss: 0.000815, epsilon: 0.010001, episode:  193\n",
      "frames: 405000, reward: 16.900000, loss: 0.002394, epsilon: 0.010001, episode:  194\n",
      "frames: 406000, reward: 16.900000, loss: 0.000869, epsilon: 0.010001, episode:  194\n",
      "frames: 407000, reward: 16.800000, loss: 0.010242, epsilon: 0.010001, episode:  195\n",
      "frames: 408000, reward: 16.800000, loss: 0.001025, epsilon: 0.010001, episode:  195\n",
      "frames: 409000, reward: 16.800000, loss: 0.003374, epsilon: 0.010001, episode:  195\n",
      "frames: 410000, reward: 16.400000, loss: 0.001225, epsilon: 0.010001, episode:  196\n",
      "frames: 411000, reward: 16.400000, loss: 0.001564, epsilon: 0.010001, episode:  196\n",
      "frames: 412000, reward: 16.800000, loss: 0.001119, epsilon: 0.010001, episode:  197\n",
      "frames: 413000, reward: 16.800000, loss: 0.001413, epsilon: 0.010001, episode:  197\n",
      "frames: 414000, reward: 17.000000, loss: 0.000888, epsilon: 0.010001, episode:  198\n",
      "frames: 415000, reward: 17.000000, loss: 0.001914, epsilon: 0.010001, episode:  198\n",
      "frames: 416000, reward: 16.800000, loss: 0.001240, epsilon: 0.010001, episode:  199\n",
      "frames: 417000, reward: 16.800000, loss: 0.001065, epsilon: 0.010001, episode:  199\n",
      "frames: 418000, reward: 16.900000, loss: 0.001271, epsilon: 0.010001, episode:  200\n",
      "frames: 419000, reward: 16.900000, loss: 0.001034, epsilon: 0.010001, episode:  200\n",
      "frames: 420000, reward: 16.900000, loss: 0.000880, epsilon: 0.010001, episode:  200\n",
      "frames: 421000, reward: 16.900000, loss: 0.001995, epsilon: 0.010001, episode:  200\n",
      "frames: 422000, reward: 15.100000, loss: 0.001248, epsilon: 0.010001, episode:  201\n",
      "frames: 423000, reward: 15.100000, loss: 0.001310, epsilon: 0.010001, episode:  201\n",
      "frames: 424000, reward: 16.000000, loss: 0.001022, epsilon: 0.010001, episode:  202\n",
      "frames: 425000, reward: 16.000000, loss: 0.000721, epsilon: 0.010001, episode:  202\n",
      "frames: 426000, reward: 16.100000, loss: 0.001021, epsilon: 0.010001, episode:  203\n",
      "frames: 427000, reward: 16.100000, loss: 0.000835, epsilon: 0.010001, episode:  203\n",
      "frames: 428000, reward: 15.600000, loss: 0.000999, epsilon: 0.010001, episode:  204\n",
      "frames: 429000, reward: 15.600000, loss: 0.000850, epsilon: 0.010001, episode:  204\n",
      "frames: 430000, reward: 15.600000, loss: 0.000917, epsilon: 0.010001, episode:  205\n",
      "frames: 431000, reward: 15.600000, loss: 0.000651, epsilon: 0.010001, episode:  205\n",
      "frames: 432000, reward: 16.100000, loss: 0.000380, epsilon: 0.010001, episode:  206\n",
      "frames: 433000, reward: 16.100000, loss: 0.001163, epsilon: 0.010001, episode:  206\n",
      "frames: 434000, reward: 16.100000, loss: 0.000454, epsilon: 0.010001, episode:  207\n",
      "frames: 435000, reward: 16.100000, loss: 0.001226, epsilon: 0.010000, episode:  207\n",
      "frames: 436000, reward: 16.000000, loss: 0.001623, epsilon: 0.010000, episode:  208\n",
      "frames: 437000, reward: 16.000000, loss: 0.000486, epsilon: 0.010000, episode:  208\n",
      "frames: 438000, reward: 16.200000, loss: 0.000383, epsilon: 0.010000, episode:  209\n",
      "frames: 439000, reward: 16.200000, loss: 0.000345, epsilon: 0.010000, episode:  209\n",
      "frames: 440000, reward: 16.400000, loss: 0.001107, epsilon: 0.010000, episode:  210\n",
      "frames: 441000, reward: 16.400000, loss: 0.000648, epsilon: 0.010000, episode:  210\n",
      "frames: 442000, reward: 18.000000, loss: 0.000854, epsilon: 0.010000, episode:  211\n",
      "frames: 443000, reward: 18.000000, loss: 0.000717, epsilon: 0.010000, episode:  211\n",
      "frames: 444000, reward: 18.000000, loss: 0.000726, epsilon: 0.010000, episode:  211\n",
      "frames: 445000, reward: 17.500000, loss: 0.000367, epsilon: 0.010000, episode:  212\n",
      "frames: 446000, reward: 17.500000, loss: 0.001384, epsilon: 0.010000, episode:  212\n",
      "frames: 447000, reward: 17.500000, loss: 0.000793, epsilon: 0.010000, episode:  212\n",
      "frames: 448000, reward: 17.500000, loss: 0.002688, epsilon: 0.010000, episode:  212\n",
      "frames: 449000, reward: 15.300000, loss: 0.001117, epsilon: 0.010000, episode:  213\n",
      "frames: 450000, reward: 15.300000, loss: 0.001239, epsilon: 0.010000, episode:  213\n",
      "frames: 451000, reward: 15.200000, loss: 0.000865, epsilon: 0.010000, episode:  214\n",
      "frames: 452000, reward: 15.200000, loss: 0.001090, epsilon: 0.010000, episode:  214\n",
      "frames: 453000, reward: 15.200000, loss: 0.000552, epsilon: 0.010000, episode:  214\n",
      "frames: 454000, reward: 15.000000, loss: 0.002539, epsilon: 0.010000, episode:  215\n",
      "frames: 455000, reward: 15.000000, loss: 0.001028, epsilon: 0.010000, episode:  215\n",
      "frames: 456000, reward: 15.000000, loss: 0.000593, epsilon: 0.010000, episode:  215\n",
      "frames: 457000, reward: 14.500000, loss: 0.001233, epsilon: 0.010000, episode:  216\n",
      "frames: 458000, reward: 14.500000, loss: 0.000761, epsilon: 0.010000, episode:  216\n",
      "frames: 459000, reward: 14.300000, loss: 0.001926, epsilon: 0.010000, episode:  217\n",
      "frames: 460000, reward: 14.300000, loss: 0.000674, epsilon: 0.010000, episode:  217\n",
      "frames: 461000, reward: 14.300000, loss: 0.001063, epsilon: 0.010000, episode:  217\n",
      "frames: 462000, reward: 13.500000, loss: 0.001374, epsilon: 0.010000, episode:  218\n",
      "frames: 463000, reward: 13.500000, loss: 0.001148, epsilon: 0.010000, episode:  218\n",
      "frames: 464000, reward: 13.300000, loss: 0.001411, epsilon: 0.010000, episode:  219\n",
      "frames: 465000, reward: 13.300000, loss: 0.001061, epsilon: 0.010000, episode:  219\n",
      "frames: 466000, reward: 13.300000, loss: 0.000929, epsilon: 0.010000, episode:  219\n",
      "frames: 467000, reward: 13.100000, loss: 0.000595, epsilon: 0.010000, episode:  220\n",
      "frames: 468000, reward: 13.100000, loss: 0.000272, epsilon: 0.010000, episode:  220\n",
      "frames: 469000, reward: 13.500000, loss: 0.001750, epsilon: 0.010000, episode:  221\n",
      "frames: 470000, reward: 13.500000, loss: 0.000602, epsilon: 0.010000, episode:  221\n",
      "frames: 471000, reward: 14.200000, loss: 0.000622, epsilon: 0.010000, episode:  222\n",
      "frames: 472000, reward: 14.200000, loss: 0.000401, epsilon: 0.010000, episode:  222\n",
      "frames: 473000, reward: 16.400000, loss: 0.000281, epsilon: 0.010000, episode:  223\n",
      "frames: 474000, reward: 16.400000, loss: 0.000320, epsilon: 0.010000, episode:  223\n",
      "frames: 475000, reward: 17.400000, loss: 0.000372, epsilon: 0.010000, episode:  224\n",
      "frames: 476000, reward: 17.400000, loss: 0.000440, epsilon: 0.010000, episode:  224\n",
      "frames: 477000, reward: 17.400000, loss: 0.000465, epsilon: 0.010000, episode:  225\n",
      "frames: 478000, reward: 17.400000, loss: 0.000833, epsilon: 0.010000, episode:  225\n",
      "frames: 479000, reward: 17.400000, loss: 0.000470, epsilon: 0.010000, episode:  225\n",
      "frames: 480000, reward: 17.700000, loss: 0.000604, epsilon: 0.010000, episode:  226\n",
      "frames: 481000, reward: 17.700000, loss: 0.000425, epsilon: 0.010000, episode:  226\n",
      "frames: 482000, reward: 18.000000, loss: 0.000873, epsilon: 0.010000, episode:  227\n",
      "frames: 483000, reward: 18.000000, loss: 0.000459, epsilon: 0.010000, episode:  227\n",
      "frames: 484000, reward: 19.100000, loss: 0.000619, epsilon: 0.010000, episode:  228\n",
      "frames: 485000, reward: 19.100000, loss: 0.000427, epsilon: 0.010000, episode:  228\n",
      "frames: 486000, reward: 19.100000, loss: 0.000465, epsilon: 0.010000, episode:  228\n",
      "frames: 487000, reward: 19.000000, loss: 0.000455, epsilon: 0.010000, episode:  229\n",
      "frames: 488000, reward: 19.000000, loss: 0.000767, epsilon: 0.010000, episode:  229\n",
      "frames: 489000, reward: 19.300000, loss: 0.000589, epsilon: 0.010000, episode:  230\n",
      "frames: 490000, reward: 19.300000, loss: 0.000462, epsilon: 0.010000, episode:  230\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 491000, reward: 19.000000, loss: 0.000982, epsilon: 0.010000, episode:  231\n",
      "frames: 492000, reward: 19.000000, loss: 0.000805, epsilon: 0.010000, episode:  231\n",
      "frames: 493000, reward: 19.000000, loss: 0.000935, epsilon: 0.010000, episode:  231\n",
      "frames: 494000, reward: 18.600000, loss: 0.000549, epsilon: 0.010000, episode:  232\n",
      "frames: 495000, reward: 18.600000, loss: 0.000582, epsilon: 0.010000, episode:  232\n",
      "frames: 496000, reward: 17.700000, loss: 0.001662, epsilon: 0.010000, episode:  233\n",
      "frames: 497000, reward: 17.700000, loss: 0.000461, epsilon: 0.010000, episode:  233\n",
      "frames: 498000, reward: 17.700000, loss: 0.000417, epsilon: 0.010000, episode:  233\n",
      "frames: 499000, reward: 17.600000, loss: 0.001035, epsilon: 0.010000, episode:  234\n",
      "frames: 500000, reward: 17.600000, loss: 0.000340, epsilon: 0.010000, episode:  234\n",
      "frames: 501000, reward: 17.600000, loss: 0.000403, epsilon: 0.010000, episode:  235\n",
      "frames: 502000, reward: 17.600000, loss: 0.000234, epsilon: 0.010000, episode:  235\n",
      "frames: 503000, reward: 17.600000, loss: 0.000477, epsilon: 0.010000, episode:  235\n",
      "frames: 504000, reward: 18.000000, loss: 0.000610, epsilon: 0.010000, episode:  236\n",
      "frames: 505000, reward: 18.000000, loss: 0.000370, epsilon: 0.010000, episode:  236\n",
      "frames: 506000, reward: 18.000000, loss: 0.000275, epsilon: 0.010000, episode:  237\n",
      "frames: 507000, reward: 18.000000, loss: 0.000292, epsilon: 0.010000, episode:  237\n",
      "frames: 508000, reward: 17.800000, loss: 0.000512, epsilon: 0.010000, episode:  238\n",
      "frames: 509000, reward: 17.800000, loss: 0.000347, epsilon: 0.010000, episode:  238\n",
      "frames: 510000, reward: 18.100000, loss: 0.000826, epsilon: 0.010000, episode:  239\n",
      "frames: 511000, reward: 18.100000, loss: 0.000447, epsilon: 0.010000, episode:  239\n",
      "frames: 512000, reward: 18.300000, loss: 0.000271, epsilon: 0.010000, episode:  240\n",
      "frames: 513000, reward: 18.300000, loss: 0.000440, epsilon: 0.010000, episode:  240\n",
      "frames: 514000, reward: 18.300000, loss: 0.000550, epsilon: 0.010000, episode:  240\n",
      "frames: 515000, reward: 18.300000, loss: 0.000599, epsilon: 0.010000, episode:  241\n",
      "frames: 516000, reward: 18.300000, loss: 0.000567, epsilon: 0.010000, episode:  241\n",
      "frames: 517000, reward: 18.600000, loss: 0.000415, epsilon: 0.010000, episode:  242\n",
      "frames: 518000, reward: 18.600000, loss: 0.000538, epsilon: 0.010000, episode:  242\n",
      "frames: 519000, reward: 19.500000, loss: 0.000526, epsilon: 0.010000, episode:  243\n",
      "frames: 520000, reward: 19.500000, loss: 0.000303, epsilon: 0.010000, episode:  243\n",
      "frames: 521000, reward: 19.500000, loss: 0.000564, epsilon: 0.010000, episode:  244\n",
      "frames: 522000, reward: 19.500000, loss: 0.000694, epsilon: 0.010000, episode:  244\n",
      "frames: 523000, reward: 19.700000, loss: 0.000529, epsilon: 0.010000, episode:  245\n",
      "frames: 524000, reward: 19.700000, loss: 0.000221, epsilon: 0.010000, episode:  245\n",
      "frames: 525000, reward: 19.700000, loss: 0.000373, epsilon: 0.010000, episode:  246\n",
      "frames: 526000, reward: 19.700000, loss: 0.000560, epsilon: 0.010000, episode:  246\n",
      "frames: 527000, reward: 19.800000, loss: 0.000215, epsilon: 0.010000, episode:  247\n",
      "frames: 528000, reward: 19.800000, loss: 0.000336, epsilon: 0.010000, episode:  247\n",
      "frames: 529000, reward: 20.000000, loss: 0.000275, epsilon: 0.010000, episode:  248\n",
      "frames: 530000, reward: 20.000000, loss: 0.000202, epsilon: 0.010000, episode:  248\n",
      "frames: 531000, reward: 20.000000, loss: 0.000444, epsilon: 0.010000, episode:  249\n",
      "frames: 532000, reward: 20.000000, loss: 0.000689, epsilon: 0.010000, episode:  249\n",
      "frames: 533000, reward: 20.000000, loss: 0.000654, epsilon: 0.010000, episode:  249\n",
      "frames: 534000, reward: 18.600000, loss: 0.000753, epsilon: 0.010000, episode:  250\n",
      "frames: 535000, reward: 18.600000, loss: 0.001007, epsilon: 0.010000, episode:  250\n",
      "frames: 536000, reward: 18.600000, loss: 0.000758, epsilon: 0.010000, episode:  250\n",
      "frames: 537000, reward: 18.700000, loss: 0.001070, epsilon: 0.010000, episode:  251\n",
      "frames: 538000, reward: 18.700000, loss: 0.000720, epsilon: 0.010000, episode:  251\n",
      "frames: 539000, reward: 18.800000, loss: 0.001153, epsilon: 0.010000, episode:  252\n",
      "frames: 540000, reward: 18.800000, loss: 0.000904, epsilon: 0.010000, episode:  252\n",
      "frames: 541000, reward: 18.000000, loss: 0.001925, epsilon: 0.010000, episode:  253\n",
      "frames: 542000, reward: 18.000000, loss: 0.004627, epsilon: 0.010000, episode:  253\n",
      "frames: 543000, reward: 18.000000, loss: 0.001474, epsilon: 0.010000, episode:  253\n",
      "frames: 544000, reward: 17.400000, loss: 0.000785, epsilon: 0.010000, episode:  254\n",
      "frames: 545000, reward: 17.400000, loss: 0.000411, epsilon: 0.010000, episode:  254\n",
      "frames: 546000, reward: 17.200000, loss: 0.000896, epsilon: 0.010000, episode:  255\n",
      "frames: 547000, reward: 17.200000, loss: 0.000589, epsilon: 0.010000, episode:  255\n",
      "frames: 548000, reward: 17.200000, loss: 0.001242, epsilon: 0.010000, episode:  255\n",
      "frames: 549000, reward: 16.700000, loss: 0.002995, epsilon: 0.010000, episode:  256\n",
      "frames: 550000, reward: 16.700000, loss: 0.001117, epsilon: 0.010000, episode:  256\n",
      "frames: 551000, reward: 16.700000, loss: 0.001747, epsilon: 0.010000, episode:  257\n",
      "frames: 552000, reward: 16.700000, loss: 0.000598, epsilon: 0.010000, episode:  257\n",
      "frames: 553000, reward: 16.600000, loss: 0.000999, epsilon: 0.010000, episode:  258\n",
      "frames: 554000, reward: 16.600000, loss: 0.000492, epsilon: 0.010000, episode:  259\n",
      "frames: 555000, reward: 16.600000, loss: 0.000709, epsilon: 0.010000, episode:  259\n",
      "frames: 556000, reward: 16.600000, loss: 0.000584, epsilon: 0.010000, episode:  259\n",
      "frames: 557000, reward: 16.600000, loss: 0.001931, epsilon: 0.010000, episode:  259\n",
      "frames: 558000, reward: 16.700000, loss: 0.000907, epsilon: 0.010000, episode:  260\n",
      "frames: 559000, reward: 16.700000, loss: 0.001467, epsilon: 0.010000, episode:  260\n",
      "frames: 560000, reward: 16.300000, loss: 0.001397, epsilon: 0.010000, episode:  261\n",
      "frames: 561000, reward: 16.300000, loss: 0.000809, epsilon: 0.010000, episode:  261\n",
      "frames: 562000, reward: 16.300000, loss: 0.000845, epsilon: 0.010000, episode:  262\n",
      "frames: 563000, reward: 16.300000, loss: 0.001908, epsilon: 0.010000, episode:  262\n",
      "frames: 564000, reward: 16.700000, loss: 0.001323, epsilon: 0.010000, episode:  263\n",
      "frames: 565000, reward: 16.700000, loss: 0.000611, epsilon: 0.010000, episode:  263\n",
      "frames: 566000, reward: 16.700000, loss: 0.000402, epsilon: 0.010000, episode:  263\n",
      "frames: 567000, reward: 17.200000, loss: 0.000596, epsilon: 0.010000, episode:  264\n",
      "frames: 568000, reward: 17.200000, loss: 0.000904, epsilon: 0.010000, episode:  264\n",
      "frames: 569000, reward: 17.000000, loss: 0.001967, epsilon: 0.010000, episode:  265\n",
      "frames: 570000, reward: 17.000000, loss: 0.000565, epsilon: 0.010000, episode:  265\n",
      "frames: 571000, reward: 17.700000, loss: 0.000255, epsilon: 0.010000, episode:  266\n",
      "frames: 572000, reward: 17.700000, loss: 0.000420, epsilon: 0.010000, episode:  266\n",
      "frames: 573000, reward: 17.500000, loss: 0.000383, epsilon: 0.010000, episode:  267\n",
      "frames: 574000, reward: 17.500000, loss: 0.000694, epsilon: 0.010000, episode:  267\n",
      "frames: 575000, reward: 17.500000, loss: 0.000735, epsilon: 0.010000, episode:  267\n",
      "frames: 576000, reward: 16.900000, loss: 0.000869, epsilon: 0.010000, episode:  268\n",
      "frames: 577000, reward: 16.900000, loss: 0.000565, epsilon: 0.010000, episode:  268\n",
      "frames: 578000, reward: 16.400000, loss: 0.001320, epsilon: 0.010000, episode:  269\n",
      "frames: 579000, reward: 16.400000, loss: 0.001346, epsilon: 0.010000, episode:  269\n",
      "frames: 580000, reward: 17.300000, loss: 0.000750, epsilon: 0.010000, episode:  270\n",
      "frames: 581000, reward: 17.300000, loss: 0.001432, epsilon: 0.010000, episode:  270\n",
      "frames: 582000, reward: 17.900000, loss: 0.000756, epsilon: 0.010000, episode:  271\n",
      "frames: 583000, reward: 17.900000, loss: 0.000636, epsilon: 0.010000, episode:  271\n",
      "frames: 584000, reward: 17.800000, loss: 0.001214, epsilon: 0.010000, episode:  272\n",
      "frames: 585000, reward: 17.800000, loss: 0.001364, epsilon: 0.010000, episode:  272\n",
      "frames: 586000, reward: 17.800000, loss: 0.000611, epsilon: 0.010000, episode:  272\n",
      "frames: 587000, reward: 17.900000, loss: 0.000420, epsilon: 0.010000, episode:  273\n",
      "frames: 588000, reward: 17.900000, loss: 0.000957, epsilon: 0.010000, episode:  274\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 589000, reward: 17.900000, loss: 0.000480, epsilon: 0.010000, episode:  274\n",
      "frames: 590000, reward: 17.900000, loss: 0.000335, epsilon: 0.010000, episode:  274\n",
      "frames: 591000, reward: 18.100000, loss: 0.001266, epsilon: 0.010000, episode:  275\n",
      "frames: 592000, reward: 18.100000, loss: 0.000297, epsilon: 0.010000, episode:  276\n",
      "frames: 593000, reward: 18.100000, loss: 0.001456, epsilon: 0.010000, episode:  276\n",
      "frames: 594000, reward: 18.100000, loss: 0.000404, epsilon: 0.010000, episode:  276\n",
      "frames: 595000, reward: 18.000000, loss: 0.000765, epsilon: 0.010000, episode:  277\n",
      "frames: 596000, reward: 18.600000, loss: 0.000588, epsilon: 0.010000, episode:  278\n",
      "frames: 597000, reward: 18.600000, loss: 0.000621, epsilon: 0.010000, episode:  278\n",
      "frames: 598000, reward: 18.600000, loss: 0.000682, epsilon: 0.010000, episode:  278\n",
      "frames: 599000, reward: 18.500000, loss: 0.001107, epsilon: 0.010000, episode:  279\n",
      "frames: 600000, reward: 18.500000, loss: 0.003180, epsilon: 0.010000, episode:  279\n",
      "frames: 601000, reward: 18.400000, loss: 0.000839, epsilon: 0.010000, episode:  280\n",
      "frames: 602000, reward: 18.400000, loss: 0.001607, epsilon: 0.010000, episode:  280\n",
      "frames: 603000, reward: 18.400000, loss: 0.001771, epsilon: 0.010000, episode:  280\n",
      "frames: 604000, reward: 16.900000, loss: 0.001114, epsilon: 0.010000, episode:  281\n",
      "frames: 605000, reward: 16.900000, loss: 0.000641, epsilon: 0.010000, episode:  281\n",
      "frames: 606000, reward: 17.100000, loss: 0.001038, epsilon: 0.010000, episode:  282\n",
      "frames: 607000, reward: 17.100000, loss: 0.000686, epsilon: 0.010000, episode:  282\n",
      "frames: 608000, reward: 17.000000, loss: 0.001068, epsilon: 0.010000, episode:  283\n",
      "frames: 609000, reward: 17.000000, loss: 0.000438, epsilon: 0.010000, episode:  283\n",
      "frames: 610000, reward: 17.100000, loss: 0.000703, epsilon: 0.010000, episode:  284\n",
      "frames: 611000, reward: 17.100000, loss: 0.000399, epsilon: 0.010000, episode:  284\n",
      "frames: 612000, reward: 17.500000, loss: 0.000635, epsilon: 0.010000, episode:  285\n",
      "frames: 613000, reward: 17.500000, loss: 0.000783, epsilon: 0.010000, episode:  285\n",
      "frames: 614000, reward: 17.500000, loss: 0.000412, epsilon: 0.010000, episode:  285\n",
      "frames: 615000, reward: 17.000000, loss: 0.000441, epsilon: 0.010000, episode:  286\n",
      "frames: 616000, reward: 17.000000, loss: 0.001013, epsilon: 0.010000, episode:  286\n",
      "frames: 617000, reward: 17.000000, loss: 0.000470, epsilon: 0.010000, episode:  287\n",
      "frames: 618000, reward: 17.000000, loss: 0.000801, epsilon: 0.010000, episode:  287\n",
      "frames: 619000, reward: 16.500000, loss: 0.000711, epsilon: 0.010000, episode:  288\n",
      "frames: 620000, reward: 16.500000, loss: 0.000338, epsilon: 0.010000, episode:  288\n",
      "frames: 621000, reward: 16.900000, loss: 0.000400, epsilon: 0.010000, episode:  289\n",
      "frames: 622000, reward: 16.900000, loss: 0.000474, epsilon: 0.010000, episode:  289\n",
      "frames: 623000, reward: 17.200000, loss: 0.001054, epsilon: 0.010000, episode:  290\n",
      "frames: 624000, reward: 17.200000, loss: 0.000473, epsilon: 0.010000, episode:  290\n",
      "frames: 625000, reward: 18.700000, loss: 0.000448, epsilon: 0.010000, episode:  291\n",
      "frames: 626000, reward: 18.700000, loss: 0.000822, epsilon: 0.010000, episode:  291\n",
      "frames: 627000, reward: 18.400000, loss: 0.000565, epsilon: 0.010000, episode:  292\n",
      "frames: 628000, reward: 18.400000, loss: 0.000354, epsilon: 0.010000, episode:  292\n",
      "frames: 629000, reward: 18.600000, loss: 0.000378, epsilon: 0.010000, episode:  293\n",
      "frames: 630000, reward: 18.600000, loss: 0.000516, epsilon: 0.010000, episode:  293\n",
      "frames: 631000, reward: 18.600000, loss: 0.000229, epsilon: 0.010000, episode:  293\n",
      "frames: 632000, reward: 18.600000, loss: 0.000636, epsilon: 0.010000, episode:  294\n",
      "frames: 633000, reward: 18.600000, loss: 0.000413, epsilon: 0.010000, episode:  294\n",
      "frames: 634000, reward: 18.400000, loss: 0.001581, epsilon: 0.010000, episode:  295\n",
      "frames: 635000, reward: 18.400000, loss: 0.000413, epsilon: 0.010000, episode:  295\n",
      "frames: 636000, reward: 18.900000, loss: 0.000650, epsilon: 0.010000, episode:  296\n",
      "frames: 637000, reward: 18.900000, loss: 0.009369, epsilon: 0.010000, episode:  296\n",
      "frames: 638000, reward: 18.900000, loss: 0.004026, epsilon: 0.010000, episode:  296\n",
      "frames: 639000, reward: 18.200000, loss: 0.000779, epsilon: 0.010000, episode:  297\n",
      "frames: 640000, reward: 18.200000, loss: 0.000434, epsilon: 0.010000, episode:  297\n",
      "frames: 641000, reward: 18.600000, loss: 0.000602, epsilon: 0.010000, episode:  298\n",
      "frames: 642000, reward: 18.600000, loss: 0.000754, epsilon: 0.010000, episode:  298\n",
      "frames: 643000, reward: 18.600000, loss: 0.000777, epsilon: 0.010000, episode:  299\n",
      "frames: 644000, reward: 18.600000, loss: 0.001618, epsilon: 0.010000, episode:  299\n",
      "frames: 645000, reward: 18.600000, loss: 0.000504, epsilon: 0.010000, episode:  299\n",
      "frames: 646000, reward: 18.300000, loss: 0.000677, epsilon: 0.010000, episode:  300\n",
      "frames: 647000, reward: 18.300000, loss: 0.000660, epsilon: 0.010000, episode:  300\n",
      "frames: 648000, reward: 18.200000, loss: 0.003337, epsilon: 0.010000, episode:  301\n",
      "frames: 649000, reward: 18.200000, loss: 0.000503, epsilon: 0.010000, episode:  301\n",
      "frames: 650000, reward: 18.200000, loss: 0.000430, epsilon: 0.010000, episode:  301\n",
      "frames: 651000, reward: 18.000000, loss: 0.000993, epsilon: 0.010000, episode:  302\n",
      "frames: 652000, reward: 18.000000, loss: 0.000344, epsilon: 0.010000, episode:  302\n",
      "frames: 653000, reward: 18.000000, loss: 0.000427, epsilon: 0.010000, episode:  303\n",
      "frames: 654000, reward: 18.000000, loss: 0.000454, epsilon: 0.010000, episode:  303\n",
      "frames: 655000, reward: 17.600000, loss: 0.000294, epsilon: 0.010000, episode:  304\n",
      "frames: 656000, reward: 17.600000, loss: 0.000478, epsilon: 0.010000, episode:  304\n",
      "frames: 657000, reward: 17.600000, loss: 0.000460, epsilon: 0.010000, episode:  305\n",
      "frames: 658000, reward: 17.600000, loss: 0.000649, epsilon: 0.010000, episode:  305\n",
      "frames: 659000, reward: 17.500000, loss: 0.000687, epsilon: 0.010000, episode:  306\n",
      "frames: 660000, reward: 17.500000, loss: 0.000315, epsilon: 0.010000, episode:  306\n",
      "frames: 661000, reward: 17.500000, loss: 0.000434, epsilon: 0.010000, episode:  306\n",
      "frames: 662000, reward: 18.200000, loss: 0.000567, epsilon: 0.010000, episode:  307\n",
      "frames: 663000, reward: 18.200000, loss: 0.000518, epsilon: 0.010000, episode:  307\n",
      "frames: 664000, reward: 18.200000, loss: 0.000961, epsilon: 0.010000, episode:  307\n",
      "frames: 665000, reward: 17.000000, loss: 0.001490, epsilon: 0.010000, episode:  308\n",
      "frames: 666000, reward: 17.000000, loss: 0.000767, epsilon: 0.010000, episode:  308\n",
      "frames: 667000, reward: 17.000000, loss: 0.000734, epsilon: 0.010000, episode:  309\n",
      "frames: 668000, reward: 17.000000, loss: 0.000489, epsilon: 0.010000, episode:  309\n",
      "frames: 669000, reward: 17.000000, loss: 0.000773, epsilon: 0.010000, episode:  309\n",
      "frames: 670000, reward: 17.000000, loss: 0.000876, epsilon: 0.010000, episode:  310\n",
      "frames: 671000, reward: 17.000000, loss: 0.000497, epsilon: 0.010000, episode:  310\n",
      "frames: 672000, reward: 17.000000, loss: 0.000583, epsilon: 0.010000, episode:  310\n",
      "frames: 673000, reward: 16.900000, loss: 0.000449, epsilon: 0.010000, episode:  311\n",
      "frames: 674000, reward: 17.400000, loss: 0.000791, epsilon: 0.010000, episode:  312\n",
      "frames: 675000, reward: 17.400000, loss: 0.000361, epsilon: 0.010000, episode:  312\n",
      "frames: 676000, reward: 17.400000, loss: 0.000500, epsilon: 0.010000, episode:  312\n",
      "frames: 677000, reward: 17.600000, loss: 0.000770, epsilon: 0.010000, episode:  313\n",
      "frames: 678000, reward: 18.100000, loss: 0.000256, epsilon: 0.010000, episode:  314\n",
      "frames: 679000, reward: 18.100000, loss: 0.000429, epsilon: 0.010000, episode:  314\n",
      "frames: 680000, reward: 18.100000, loss: 0.000320, epsilon: 0.010000, episode:  315\n",
      "frames: 681000, reward: 18.100000, loss: 0.000262, epsilon: 0.010000, episode:  315\n",
      "frames: 682000, reward: 18.000000, loss: 0.000408, epsilon: 0.010000, episode:  316\n",
      "frames: 683000, reward: 18.000000, loss: 0.000583, epsilon: 0.010000, episode:  316\n",
      "frames: 684000, reward: 18.300000, loss: 0.000535, epsilon: 0.010000, episode:  317\n",
      "frames: 685000, reward: 18.300000, loss: 0.000885, epsilon: 0.010000, episode:  317\n",
      "frames: 686000, reward: 19.700000, loss: 0.000517, epsilon: 0.010000, episode:  318\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 687000, reward: 19.700000, loss: 0.000477, epsilon: 0.010000, episode:  318\n",
      "frames: 688000, reward: 19.900000, loss: 0.000274, epsilon: 0.010000, episode:  319\n",
      "frames: 689000, reward: 19.900000, loss: 0.000357, epsilon: 0.010000, episode:  319\n",
      "frames: 690000, reward: 20.400000, loss: 0.000718, epsilon: 0.010000, episode:  320\n",
      "frames: 691000, reward: 20.400000, loss: 0.000328, epsilon: 0.010000, episode:  320\n",
      "frames: 692000, reward: 20.300000, loss: 0.000715, epsilon: 0.010000, episode:  321\n",
      "frames: 693000, reward: 20.300000, loss: 0.000307, epsilon: 0.010000, episode:  321\n",
      "frames: 694000, reward: 20.300000, loss: 0.001243, epsilon: 0.010000, episode:  321\n",
      "frames: 695000, reward: 19.100000, loss: 0.000442, epsilon: 0.010000, episode:  322\n",
      "frames: 696000, reward: 19.100000, loss: 0.000897, epsilon: 0.010000, episode:  322\n",
      "frames: 697000, reward: 18.700000, loss: 0.000699, epsilon: 0.010000, episode:  323\n",
      "frames: 698000, reward: 18.700000, loss: 0.000323, epsilon: 0.010000, episode:  323\n",
      "frames: 699000, reward: 18.700000, loss: 0.000706, epsilon: 0.010000, episode:  323\n",
      "frames: 700000, reward: 16.700000, loss: 0.000574, epsilon: 0.010000, episode:  324\n",
      "frames: 701000, reward: 16.700000, loss: 0.001256, epsilon: 0.010000, episode:  324\n",
      "frames: 702000, reward: 16.700000, loss: 0.001323, epsilon: 0.010000, episode:  324\n",
      "frames: 703000, reward: 16.000000, loss: 0.001632, epsilon: 0.010000, episode:  325\n",
      "frames: 704000, reward: 16.000000, loss: 0.001884, epsilon: 0.010000, episode:  325\n",
      "frames: 705000, reward: 16.000000, loss: 0.001589, epsilon: 0.010000, episode:  326\n",
      "frames: 706000, reward: 16.000000, loss: 0.001461, epsilon: 0.010000, episode:  326\n",
      "frames: 707000, reward: 15.900000, loss: 0.000695, epsilon: 0.010000, episode:  327\n",
      "frames: 708000, reward: 15.900000, loss: 0.000297, epsilon: 0.010000, episode:  327\n",
      "frames: 709000, reward: 15.900000, loss: 0.001004, epsilon: 0.010000, episode:  328\n",
      "frames: 710000, reward: 15.900000, loss: 0.000361, epsilon: 0.010000, episode:  328\n",
      "frames: 711000, reward: 15.700000, loss: 0.000308, epsilon: 0.010000, episode:  329\n",
      "frames: 712000, reward: 15.700000, loss: 0.000357, epsilon: 0.010000, episode:  329\n",
      "frames: 713000, reward: 15.700000, loss: 0.000593, epsilon: 0.010000, episode:  330\n",
      "frames: 714000, reward: 15.700000, loss: 0.000383, epsilon: 0.010000, episode:  330\n",
      "frames: 715000, reward: 15.700000, loss: 0.000281, epsilon: 0.010000, episode:  330\n",
      "frames: 716000, reward: 15.800000, loss: 0.000487, epsilon: 0.010000, episode:  331\n",
      "frames: 717000, reward: 15.800000, loss: 0.001110, epsilon: 0.010000, episode:  331\n",
      "frames: 718000, reward: 16.800000, loss: 0.001685, epsilon: 0.010000, episode:  332\n",
      "frames: 719000, reward: 16.800000, loss: 0.000428, epsilon: 0.010000, episode:  332\n",
      "frames: 720000, reward: 16.900000, loss: 0.000727, epsilon: 0.010000, episode:  333\n",
      "frames: 721000, reward: 16.900000, loss: 0.000747, epsilon: 0.010000, episode:  333\n",
      "frames: 722000, reward: 16.900000, loss: 0.000230, epsilon: 0.010000, episode:  333\n",
      "frames: 723000, reward: 18.600000, loss: 0.000653, epsilon: 0.010000, episode:  334\n",
      "frames: 724000, reward: 19.500000, loss: 0.000626, epsilon: 0.010000, episode:  335\n",
      "frames: 725000, reward: 19.500000, loss: 0.001101, epsilon: 0.010000, episode:  335\n",
      "frames: 726000, reward: 19.500000, loss: 0.001068, epsilon: 0.010000, episode:  335\n",
      "frames: 727000, reward: 19.500000, loss: 0.002834, epsilon: 0.010000, episode:  335\n",
      "frames: 728000, reward: 18.100000, loss: 0.000574, epsilon: 0.010000, episode:  336\n",
      "frames: 729000, reward: 18.100000, loss: 0.000372, epsilon: 0.010000, episode:  336\n",
      "frames: 730000, reward: 17.900000, loss: 0.000738, epsilon: 0.010000, episode:  337\n",
      "frames: 731000, reward: 17.900000, loss: 0.000480, epsilon: 0.010000, episode:  337\n",
      "frames: 732000, reward: 17.400000, loss: 0.000575, epsilon: 0.010000, episode:  338\n",
      "frames: 733000, reward: 17.400000, loss: 0.000456, epsilon: 0.010000, episode:  338\n",
      "frames: 734000, reward: 17.500000, loss: 0.001077, epsilon: 0.010000, episode:  339\n",
      "frames: 735000, reward: 17.500000, loss: 0.003600, epsilon: 0.010000, episode:  339\n",
      "frames: 736000, reward: 16.400000, loss: 0.000815, epsilon: 0.010000, episode:  340\n",
      "frames: 737000, reward: 16.400000, loss: 0.000644, epsilon: 0.010000, episode:  340\n",
      "frames: 738000, reward: 16.400000, loss: 0.000637, epsilon: 0.010000, episode:  341\n",
      "frames: 739000, reward: 16.400000, loss: 0.000475, epsilon: 0.010000, episode:  341\n",
      "frames: 740000, reward: 16.100000, loss: 0.000464, epsilon: 0.010000, episode:  342\n",
      "frames: 741000, reward: 16.100000, loss: 0.000841, epsilon: 0.010000, episode:  342\n",
      "frames: 742000, reward: 16.200000, loss: 0.000993, epsilon: 0.010000, episode:  343\n",
      "frames: 743000, reward: 16.200000, loss: 0.000538, epsilon: 0.010000, episode:  343\n",
      "frames: 744000, reward: 15.800000, loss: 0.000577, epsilon: 0.010000, episode:  344\n",
      "frames: 745000, reward: 15.800000, loss: 0.000940, epsilon: 0.010000, episode:  344\n",
      "frames: 746000, reward: 15.800000, loss: 0.000500, epsilon: 0.010000, episode:  345\n",
      "frames: 747000, reward: 15.800000, loss: 0.000882, epsilon: 0.010000, episode:  345\n",
      "frames: 748000, reward: 17.100000, loss: 0.000605, epsilon: 0.010000, episode:  346\n",
      "frames: 749000, reward: 17.100000, loss: 0.000375, epsilon: 0.010000, episode:  346\n",
      "frames: 750000, reward: 17.100000, loss: 0.031801, epsilon: 0.010000, episode:  347\n",
      "frames: 751000, reward: 17.100000, loss: 0.000517, epsilon: 0.010000, episode:  347\n",
      "frames: 752000, reward: 17.100000, loss: 0.000304, epsilon: 0.010000, episode:  347\n",
      "frames: 753000, reward: 17.400000, loss: 0.000918, epsilon: 0.010000, episode:  348\n",
      "frames: 754000, reward: 17.400000, loss: 0.000706, epsilon: 0.010000, episode:  348\n",
      "frames: 755000, reward: 17.200000, loss: 0.000909, epsilon: 0.010000, episode:  349\n",
      "frames: 756000, reward: 17.200000, loss: 0.000943, epsilon: 0.010000, episode:  349\n",
      "frames: 757000, reward: 17.700000, loss: 0.000795, epsilon: 0.010000, episode:  350\n",
      "frames: 758000, reward: 17.700000, loss: 0.004035, epsilon: 0.010000, episode:  350\n",
      "frames: 759000, reward: 17.700000, loss: 0.000634, epsilon: 0.010000, episode:  350\n",
      "frames: 760000, reward: 17.700000, loss: 0.001423, epsilon: 0.010000, episode:  350\n",
      "frames: 761000, reward: 16.500000, loss: 0.000582, epsilon: 0.010000, episode:  351\n",
      "frames: 762000, reward: 16.500000, loss: 0.000958, epsilon: 0.010000, episode:  351\n",
      "frames: 763000, reward: 17.000000, loss: 0.000974, epsilon: 0.010000, episode:  352\n",
      "frames: 764000, reward: 17.000000, loss: 0.000822, epsilon: 0.010000, episode:  352\n",
      "frames: 765000, reward: 15.900000, loss: 0.001423, epsilon: 0.010000, episode:  353\n",
      "frames: 766000, reward: 15.900000, loss: 0.001139, epsilon: 0.010000, episode:  353\n",
      "frames: 767000, reward: 15.900000, loss: 0.001179, epsilon: 0.010000, episode:  353\n",
      "frames: 768000, reward: 16.300000, loss: 0.001068, epsilon: 0.010000, episode:  354\n",
      "frames: 769000, reward: 16.300000, loss: 0.000882, epsilon: 0.010000, episode:  354\n",
      "frames: 770000, reward: 16.200000, loss: 0.000649, epsilon: 0.010000, episode:  355\n",
      "frames: 771000, reward: 16.200000, loss: 0.000857, epsilon: 0.010000, episode:  355\n",
      "frames: 772000, reward: 16.100000, loss: 0.001522, epsilon: 0.010000, episode:  356\n",
      "frames: 773000, reward: 16.100000, loss: 0.000928, epsilon: 0.010000, episode:  356\n",
      "frames: 774000, reward: 15.900000, loss: 0.000802, epsilon: 0.010000, episode:  357\n",
      "frames: 775000, reward: 15.900000, loss: 0.000754, epsilon: 0.010000, episode:  357\n",
      "frames: 776000, reward: 15.900000, loss: 0.000686, epsilon: 0.010000, episode:  357\n",
      "frames: 777000, reward: 15.600000, loss: 0.000721, epsilon: 0.010000, episode:  358\n",
      "frames: 778000, reward: 15.600000, loss: 0.001002, epsilon: 0.010000, episode:  358\n",
      "frames: 779000, reward: 15.900000, loss: 0.000433, epsilon: 0.010000, episode:  359\n",
      "frames: 780000, reward: 15.900000, loss: 0.000411, epsilon: 0.010000, episode:  359\n",
      "frames: 781000, reward: 16.500000, loss: 0.000480, epsilon: 0.010000, episode:  360\n",
      "frames: 782000, reward: 16.500000, loss: 0.000379, epsilon: 0.010000, episode:  360\n",
      "frames: 783000, reward: 17.600000, loss: 0.000483, epsilon: 0.010000, episode:  361\n",
      "frames: 784000, reward: 17.600000, loss: 0.000482, epsilon: 0.010000, episode:  361\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 785000, reward: 17.400000, loss: 0.000470, epsilon: 0.010000, episode:  362\n",
      "frames: 786000, reward: 17.400000, loss: 0.000259, epsilon: 0.010000, episode:  362\n",
      "frames: 787000, reward: 18.700000, loss: 0.000514, epsilon: 0.010000, episode:  363\n",
      "frames: 788000, reward: 18.700000, loss: 0.000359, epsilon: 0.010000, episode:  363\n",
      "frames: 789000, reward: 18.700000, loss: 0.000644, epsilon: 0.010000, episode:  364\n",
      "frames: 790000, reward: 18.700000, loss: 0.000493, epsilon: 0.010000, episode:  364\n",
      "frames: 791000, reward: 18.700000, loss: 0.000648, epsilon: 0.010000, episode:  364\n",
      "frames: 792000, reward: 18.700000, loss: 0.000425, epsilon: 0.010000, episode:  365\n",
      "frames: 793000, reward: 18.700000, loss: 0.000274, epsilon: 0.010000, episode:  365\n",
      "frames: 794000, reward: 19.000000, loss: 0.000641, epsilon: 0.010000, episode:  366\n",
      "frames: 795000, reward: 19.000000, loss: 0.000163, epsilon: 0.010000, episode:  366\n",
      "frames: 796000, reward: 19.300000, loss: 0.000527, epsilon: 0.010000, episode:  367\n",
      "frames: 797000, reward: 19.300000, loss: 0.000594, epsilon: 0.010000, episode:  367\n",
      "frames: 798000, reward: 19.500000, loss: 0.000742, epsilon: 0.010000, episode:  368\n",
      "frames: 799000, reward: 19.500000, loss: 0.000604, epsilon: 0.010000, episode:  368\n",
      "frames: 800000, reward: 18.800000, loss: 0.001084, epsilon: 0.010000, episode:  369\n",
      "frames: 801000, reward: 18.800000, loss: 0.000729, epsilon: 0.010000, episode:  369\n",
      "frames: 802000, reward: 18.800000, loss: 0.000408, epsilon: 0.010000, episode:  369\n",
      "frames: 803000, reward: 18.800000, loss: 0.000910, epsilon: 0.010000, episode:  369\n",
      "frames: 804000, reward: 18.800000, loss: 0.000613, epsilon: 0.010000, episode:  369\n",
      "frames: 805000, reward: 18.300000, loss: 0.001197, epsilon: 0.010000, episode:  370\n",
      "frames: 806000, reward: 18.300000, loss: 0.000373, epsilon: 0.010000, episode:  370\n",
      "frames: 807000, reward: 18.300000, loss: 0.000469, epsilon: 0.010000, episode:  370\n",
      "frames: 808000, reward: 18.300000, loss: 0.001285, epsilon: 0.010000, episode:  371\n",
      "frames: 809000, reward: 18.300000, loss: 0.001555, epsilon: 0.010000, episode:  371\n",
      "frames: 810000, reward: 18.200000, loss: 0.000637, epsilon: 0.010000, episode:  372\n",
      "frames: 811000, reward: 18.200000, loss: 0.000541, epsilon: 0.010000, episode:  372\n",
      "frames: 812000, reward: 17.700000, loss: 0.000873, epsilon: 0.010000, episode:  373\n",
      "frames: 813000, reward: 17.700000, loss: 0.001132, epsilon: 0.010000, episode:  373\n",
      "frames: 814000, reward: 17.700000, loss: 0.000695, epsilon: 0.010000, episode:  373\n",
      "frames: 815000, reward: 17.700000, loss: 0.000761, epsilon: 0.010000, episode:  374\n",
      "frames: 816000, reward: 17.700000, loss: 0.001170, epsilon: 0.010000, episode:  374\n",
      "frames: 817000, reward: 17.300000, loss: 0.003047, epsilon: 0.010000, episode:  375\n",
      "frames: 818000, reward: 17.300000, loss: 0.004219, epsilon: 0.010000, episode:  375\n",
      "frames: 819000, reward: 17.100000, loss: 0.001212, epsilon: 0.010000, episode:  376\n",
      "frames: 820000, reward: 17.100000, loss: 0.000553, epsilon: 0.010000, episode:  376\n",
      "frames: 821000, reward: 17.100000, loss: 0.000193, epsilon: 0.010000, episode:  376\n",
      "frames: 822000, reward: 17.200000, loss: 0.000551, epsilon: 0.010000, episode:  377\n",
      "frames: 823000, reward: 17.200000, loss: 0.000405, epsilon: 0.010000, episode:  377\n",
      "frames: 824000, reward: 17.200000, loss: 0.000685, epsilon: 0.010000, episode:  377\n",
      "frames: 825000, reward: 17.000000, loss: 0.000593, epsilon: 0.010000, episode:  378\n",
      "frames: 826000, reward: 17.000000, loss: 0.000433, epsilon: 0.010000, episode:  378\n",
      "frames: 827000, reward: 17.600000, loss: 0.000528, epsilon: 0.010000, episode:  379\n",
      "frames: 828000, reward: 17.600000, loss: 0.000679, epsilon: 0.010000, episode:  379\n",
      "frames: 829000, reward: 17.900000, loss: 0.000446, epsilon: 0.010000, episode:  380\n",
      "frames: 830000, reward: 17.900000, loss: 0.000459, epsilon: 0.010000, episode:  380\n",
      "frames: 831000, reward: 17.900000, loss: 0.000881, epsilon: 0.010000, episode:  380\n",
      "frames: 832000, reward: 16.900000, loss: 0.001016, epsilon: 0.010000, episode:  381\n",
      "frames: 833000, reward: 16.900000, loss: 0.001428, epsilon: 0.010000, episode:  381\n",
      "frames: 834000, reward: 16.900000, loss: 0.000643, epsilon: 0.010000, episode:  381\n",
      "frames: 835000, reward: 16.800000, loss: 0.000788, epsilon: 0.010000, episode:  382\n",
      "frames: 836000, reward: 16.800000, loss: 0.000755, epsilon: 0.010000, episode:  382\n",
      "frames: 837000, reward: 17.100000, loss: 0.000968, epsilon: 0.010000, episode:  383\n",
      "frames: 838000, reward: 17.100000, loss: 0.001154, epsilon: 0.010000, episode:  383\n",
      "frames: 839000, reward: 17.100000, loss: 0.000534, epsilon: 0.010000, episode:  383\n",
      "frames: 840000, reward: 17.300000, loss: 0.000662, epsilon: 0.010000, episode:  384\n",
      "frames: 841000, reward: 17.300000, loss: 0.000327, epsilon: 0.010000, episode:  384\n",
      "frames: 842000, reward: 17.800000, loss: 0.000502, epsilon: 0.010000, episode:  385\n",
      "frames: 843000, reward: 17.800000, loss: 0.000440, epsilon: 0.010000, episode:  385\n",
      "frames: 844000, reward: 17.900000, loss: 0.000302, epsilon: 0.010000, episode:  386\n",
      "frames: 845000, reward: 17.900000, loss: 0.000574, epsilon: 0.010000, episode:  386\n",
      "frames: 846000, reward: 17.800000, loss: 0.002205, epsilon: 0.010000, episode:  387\n",
      "frames: 847000, reward: 17.800000, loss: 0.000501, epsilon: 0.010000, episode:  387\n",
      "frames: 848000, reward: 17.400000, loss: 0.000804, epsilon: 0.010000, episode:  388\n",
      "frames: 849000, reward: 17.400000, loss: 0.000365, epsilon: 0.010000, episode:  388\n",
      "frames: 850000, reward: 17.400000, loss: 0.001373, epsilon: 0.010000, episode:  388\n",
      "frames: 851000, reward: 17.200000, loss: 0.000938, epsilon: 0.010000, episode:  389\n",
      "frames: 852000, reward: 17.100000, loss: 0.002567, epsilon: 0.010000, episode:  390\n",
      "frames: 853000, reward: 17.100000, loss: 0.000621, epsilon: 0.010000, episode:  390\n",
      "frames: 854000, reward: 18.400000, loss: 0.000871, epsilon: 0.010000, episode:  391\n",
      "frames: 855000, reward: 18.400000, loss: 0.000490, epsilon: 0.010000, episode:  391\n",
      "frames: 856000, reward: 18.400000, loss: 0.000451, epsilon: 0.010000, episode:  391\n",
      "frames: 857000, reward: 18.800000, loss: 0.000520, epsilon: 0.010000, episode:  392\n",
      "frames: 858000, reward: 18.800000, loss: 0.000456, epsilon: 0.010000, episode:  392\n",
      "frames: 859000, reward: 18.500000, loss: 0.000504, epsilon: 0.010000, episode:  393\n",
      "frames: 860000, reward: 18.500000, loss: 0.000797, epsilon: 0.010000, episode:  393\n",
      "frames: 861000, reward: 18.500000, loss: 0.000505, epsilon: 0.010000, episode:  394\n",
      "frames: 862000, reward: 18.500000, loss: 0.000185, epsilon: 0.010000, episode:  394\n",
      "frames: 863000, reward: 18.000000, loss: 0.000788, epsilon: 0.010000, episode:  395\n",
      "frames: 864000, reward: 18.000000, loss: 0.001788, epsilon: 0.010000, episode:  395\n",
      "frames: 865000, reward: 18.000000, loss: 0.000631, epsilon: 0.010000, episode:  395\n",
      "frames: 866000, reward: 17.600000, loss: 0.000307, epsilon: 0.010000, episode:  396\n",
      "frames: 867000, reward: 17.600000, loss: 0.000400, epsilon: 0.010000, episode:  396\n",
      "frames: 868000, reward: 17.400000, loss: 0.001098, epsilon: 0.010000, episode:  397\n",
      "frames: 869000, reward: 17.400000, loss: 0.000694, epsilon: 0.010000, episode:  397\n",
      "frames: 870000, reward: 18.100000, loss: 0.000388, epsilon: 0.010000, episode:  398\n",
      "frames: 871000, reward: 18.100000, loss: 0.000692, epsilon: 0.010000, episode:  398\n",
      "frames: 872000, reward: 18.100000, loss: 0.000708, epsilon: 0.010000, episode:  398\n",
      "frames: 873000, reward: 17.700000, loss: 0.000630, epsilon: 0.010000, episode:  399\n",
      "frames: 874000, reward: 17.700000, loss: 0.000367, epsilon: 0.010000, episode:  399\n",
      "frames: 875000, reward: 17.800000, loss: 0.000399, epsilon: 0.010000, episode:  400\n",
      "frames: 876000, reward: 17.800000, loss: 0.000844, epsilon: 0.010000, episode:  400\n",
      "frames: 877000, reward: 17.800000, loss: 0.000794, epsilon: 0.010000, episode:  400\n",
      "frames: 878000, reward: 17.100000, loss: 0.000535, epsilon: 0.010000, episode:  401\n",
      "frames: 879000, reward: 17.100000, loss: 0.001223, epsilon: 0.010000, episode:  401\n",
      "frames: 880000, reward: 16.800000, loss: 0.000904, epsilon: 0.010000, episode:  402\n",
      "frames: 881000, reward: 16.800000, loss: 0.000554, epsilon: 0.010000, episode:  402\n",
      "frames: 882000, reward: 16.800000, loss: 0.000576, epsilon: 0.010000, episode:  402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 883000, reward: 16.600000, loss: 0.000432, epsilon: 0.010000, episode:  403\n",
      "frames: 884000, reward: 16.600000, loss: 0.000548, epsilon: 0.010000, episode:  403\n",
      "frames: 885000, reward: 16.600000, loss: 0.000909, epsilon: 0.010000, episode:  403\n",
      "frames: 886000, reward: 15.600000, loss: 0.001002, epsilon: 0.010000, episode:  404\n",
      "frames: 887000, reward: 15.600000, loss: 0.000354, epsilon: 0.010000, episode:  404\n",
      "frames: 888000, reward: 15.600000, loss: 0.000392, epsilon: 0.010000, episode:  404\n",
      "frames: 889000, reward: 16.000000, loss: 0.002530, epsilon: 0.010000, episode:  405\n",
      "frames: 890000, reward: 16.000000, loss: 0.000766, epsilon: 0.010000, episode:  405\n",
      "frames: 891000, reward: 16.300000, loss: 0.000254, epsilon: 0.010000, episode:  406\n",
      "frames: 892000, reward: 16.300000, loss: 0.000528, epsilon: 0.010000, episode:  406\n",
      "frames: 893000, reward: 16.700000, loss: 0.002200, epsilon: 0.010000, episode:  407\n",
      "frames: 894000, reward: 16.600000, loss: 0.002161, epsilon: 0.010000, episode:  408\n",
      "frames: 895000, reward: 16.600000, loss: 0.000234, epsilon: 0.010000, episode:  408\n",
      "frames: 896000, reward: 17.100000, loss: 0.001046, epsilon: 0.010000, episode:  409\n",
      "frames: 897000, reward: 17.100000, loss: 0.000643, epsilon: 0.010000, episode:  409\n",
      "frames: 898000, reward: 17.000000, loss: 0.000645, epsilon: 0.010000, episode:  410\n",
      "frames: 899000, reward: 17.000000, loss: 0.001416, epsilon: 0.010000, episode:  410\n",
      "frames: 900000, reward: 17.000000, loss: 0.000524, epsilon: 0.010000, episode:  410\n",
      "frames: 901000, reward: 17.600000, loss: 0.000529, epsilon: 0.010000, episode:  411\n",
      "frames: 902000, reward: 17.600000, loss: 0.000541, epsilon: 0.010000, episode:  411\n",
      "frames: 903000, reward: 17.600000, loss: 0.000358, epsilon: 0.010000, episode:  412\n",
      "frames: 904000, reward: 17.600000, loss: 0.000392, epsilon: 0.010000, episode:  412\n",
      "frames: 905000, reward: 18.200000, loss: 0.000365, epsilon: 0.010000, episode:  413\n",
      "frames: 906000, reward: 18.200000, loss: 0.001361, epsilon: 0.010000, episode:  413\n",
      "frames: 907000, reward: 18.200000, loss: 0.000483, epsilon: 0.010000, episode:  413\n",
      "frames: 908000, reward: 18.500000, loss: 0.000767, epsilon: 0.010000, episode:  414\n",
      "frames: 909000, reward: 18.500000, loss: 0.000358, epsilon: 0.010000, episode:  414\n",
      "frames: 910000, reward: 18.500000, loss: 0.000433, epsilon: 0.010000, episode:  414\n",
      "frames: 911000, reward: 18.000000, loss: 0.001109, epsilon: 0.010000, episode:  415\n",
      "frames: 912000, reward: 18.000000, loss: 0.000735, epsilon: 0.010000, episode:  415\n",
      "frames: 913000, reward: 18.000000, loss: 0.001904, epsilon: 0.010000, episode:  415\n",
      "frames: 914000, reward: 17.500000, loss: 0.000913, epsilon: 0.010000, episode:  416\n",
      "frames: 915000, reward: 17.500000, loss: 0.000795, epsilon: 0.010000, episode:  416\n",
      "frames: 916000, reward: 17.400000, loss: 0.000867, epsilon: 0.010000, episode:  417\n",
      "frames: 917000, reward: 17.400000, loss: 0.002171, epsilon: 0.010000, episode:  417\n",
      "frames: 918000, reward: 17.000000, loss: 0.000493, epsilon: 0.010000, episode:  418\n",
      "frames: 919000, reward: 17.000000, loss: 0.000566, epsilon: 0.010000, episode:  418\n",
      "frames: 920000, reward: 17.000000, loss: 0.000752, epsilon: 0.010000, episode:  418\n",
      "frames: 921000, reward: 16.600000, loss: 0.000938, epsilon: 0.010000, episode:  419\n",
      "frames: 922000, reward: 16.600000, loss: 0.001133, epsilon: 0.010000, episode:  419\n",
      "frames: 923000, reward: 16.600000, loss: 0.003707, epsilon: 0.010000, episode:  419\n",
      "frames: 924000, reward: 16.100000, loss: 0.000687, epsilon: 0.010000, episode:  420\n",
      "frames: 925000, reward: 16.100000, loss: 0.000660, epsilon: 0.010000, episode:  420\n",
      "frames: 926000, reward: 16.100000, loss: 0.000544, epsilon: 0.010000, episode:  421\n",
      "frames: 927000, reward: 16.100000, loss: 0.000333, epsilon: 0.010000, episode:  421\n",
      "frames: 928000, reward: 16.400000, loss: 0.000577, epsilon: 0.010000, episode:  422\n",
      "frames: 929000, reward: 16.400000, loss: 0.000594, epsilon: 0.010000, episode:  422\n",
      "frames: 930000, reward: 16.000000, loss: 0.001629, epsilon: 0.010000, episode:  423\n",
      "frames: 931000, reward: 16.000000, loss: 0.000281, epsilon: 0.010000, episode:  423\n",
      "frames: 932000, reward: 16.800000, loss: 0.000380, epsilon: 0.010000, episode:  424\n",
      "frames: 933000, reward: 16.800000, loss: 0.000354, epsilon: 0.010000, episode:  424\n",
      "frames: 934000, reward: 17.400000, loss: 0.000374, epsilon: 0.010000, episode:  425\n",
      "frames: 935000, reward: 17.400000, loss: 0.000224, epsilon: 0.010000, episode:  425\n",
      "frames: 936000, reward: 18.200000, loss: 0.000340, epsilon: 0.010000, episode:  426\n",
      "frames: 937000, reward: 18.200000, loss: 0.000378, epsilon: 0.010000, episode:  426\n",
      "frames: 938000, reward: 18.200000, loss: 0.000451, epsilon: 0.010000, episode:  427\n",
      "frames: 939000, reward: 18.200000, loss: 0.000882, epsilon: 0.010000, episode:  427\n",
      "frames: 940000, reward: 18.400000, loss: 0.001531, epsilon: 0.010000, episode:  428\n",
      "frames: 941000, reward: 18.400000, loss: 0.000471, epsilon: 0.010000, episode:  428\n",
      "frames: 942000, reward: 18.400000, loss: 0.001272, epsilon: 0.010000, episode:  428\n",
      "frames: 943000, reward: 18.500000, loss: 0.000307, epsilon: 0.010000, episode:  429\n",
      "frames: 944000, reward: 18.500000, loss: 0.000344, epsilon: 0.010000, episode:  429\n",
      "frames: 945000, reward: 19.300000, loss: 0.000410, epsilon: 0.010000, episode:  430\n",
      "frames: 946000, reward: 19.300000, loss: 0.000638, epsilon: 0.010000, episode:  430\n",
      "frames: 947000, reward: 19.100000, loss: 0.001919, epsilon: 0.010000, episode:  431\n",
      "frames: 948000, reward: 19.100000, loss: 0.000258, epsilon: 0.010000, episode:  431\n",
      "frames: 949000, reward: 18.700000, loss: 0.000838, epsilon: 0.010000, episode:  432\n",
      "frames: 950000, reward: 18.700000, loss: 0.000169, epsilon: 0.010000, episode:  432\n",
      "frames: 951000, reward: 19.100000, loss: 0.001062, epsilon: 0.010000, episode:  433\n",
      "frames: 952000, reward: 19.100000, loss: 0.000466, epsilon: 0.010000, episode:  433\n",
      "frames: 953000, reward: 19.000000, loss: 0.000857, epsilon: 0.010000, episode:  434\n",
      "frames: 954000, reward: 19.000000, loss: 0.000393, epsilon: 0.010000, episode:  434\n",
      "frames: 955000, reward: 19.000000, loss: 0.000257, epsilon: 0.010000, episode:  435\n",
      "frames: 956000, reward: 19.000000, loss: 0.000440, epsilon: 0.010000, episode:  435\n",
      "frames: 957000, reward: 19.000000, loss: 0.000340, epsilon: 0.010000, episode:  436\n",
      "frames: 958000, reward: 19.000000, loss: 0.000211, epsilon: 0.010000, episode:  436\n",
      "frames: 959000, reward: 19.100000, loss: 0.000188, epsilon: 0.010000, episode:  437\n",
      "frames: 960000, reward: 19.100000, loss: 0.000094, epsilon: 0.010000, episode:  437\n",
      "frames: 961000, reward: 19.600000, loss: 0.000757, epsilon: 0.010000, episode:  438\n",
      "frames: 962000, reward: 19.600000, loss: 0.000156, epsilon: 0.010000, episode:  438\n",
      "frames: 963000, reward: 19.600000, loss: 0.002270, epsilon: 0.010000, episode:  439\n",
      "frames: 964000, reward: 19.600000, loss: 0.000277, epsilon: 0.010000, episode:  439\n",
      "frames: 965000, reward: 19.000000, loss: 0.000542, epsilon: 0.010000, episode:  440\n",
      "frames: 966000, reward: 19.000000, loss: 0.000215, epsilon: 0.010000, episode:  440\n",
      "frames: 967000, reward: 19.300000, loss: 0.000628, epsilon: 0.010000, episode:  441\n",
      "frames: 968000, reward: 19.500000, loss: 0.000469, epsilon: 0.010000, episode:  442\n",
      "frames: 969000, reward: 19.500000, loss: 0.001744, epsilon: 0.010000, episode:  442\n",
      "frames: 970000, reward: 19.500000, loss: 0.000285, epsilon: 0.010000, episode:  442\n",
      "frames: 971000, reward: 19.100000, loss: 0.001441, epsilon: 0.010000, episode:  443\n",
      "frames: 972000, reward: 15.000000, loss: 0.000831, epsilon: 0.010000, episode:  444\n",
      "frames: 973000, reward: 15.000000, loss: 0.001458, epsilon: 0.010000, episode:  444\n",
      "frames: 974000, reward: 14.100000, loss: 0.000880, epsilon: 0.010000, episode:  445\n",
      "frames: 975000, reward: 14.100000, loss: 0.000613, epsilon: 0.010000, episode:  445\n",
      "frames: 976000, reward: 14.100000, loss: 0.001905, epsilon: 0.010000, episode:  445\n",
      "frames: 977000, reward: 13.600000, loss: 0.001352, epsilon: 0.010000, episode:  446\n",
      "frames: 978000, reward: 13.600000, loss: 0.000628, epsilon: 0.010000, episode:  447\n",
      "frames: 979000, reward: 13.600000, loss: 0.000398, epsilon: 0.010000, episode:  447\n",
      "frames: 980000, reward: 13.600000, loss: 0.000292, epsilon: 0.010000, episode:  448\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 981000, reward: 13.600000, loss: 0.000859, epsilon: 0.010000, episode:  448\n",
      "frames: 982000, reward: 13.400000, loss: 0.000753, epsilon: 0.010000, episode:  449\n",
      "frames: 983000, reward: 13.400000, loss: 0.000417, epsilon: 0.010000, episode:  449\n",
      "frames: 984000, reward: 14.000000, loss: 0.000243, epsilon: 0.010000, episode:  450\n",
      "frames: 985000, reward: 14.000000, loss: 0.000363, epsilon: 0.010000, episode:  450\n",
      "frames: 986000, reward: 14.000000, loss: 0.000976, epsilon: 0.010000, episode:  451\n",
      "frames: 987000, reward: 14.000000, loss: 0.000531, epsilon: 0.010000, episode:  451\n",
      "frames: 988000, reward: 14.000000, loss: 0.000593, epsilon: 0.010000, episode:  451\n",
      "frames: 989000, reward: 13.700000, loss: 0.000334, epsilon: 0.010000, episode:  452\n",
      "frames: 990000, reward: 13.700000, loss: 0.000803, epsilon: 0.010000, episode:  452\n",
      "frames: 991000, reward: 13.700000, loss: 0.000675, epsilon: 0.010000, episode:  453\n",
      "frames: 992000, reward: 13.700000, loss: 0.000878, epsilon: 0.010000, episode:  453\n",
      "frames: 993000, reward: 13.700000, loss: 0.000696, epsilon: 0.010000, episode:  453\n",
      "frames: 994000, reward: 17.200000, loss: 0.000610, epsilon: 0.010000, episode:  454\n",
      "frames: 995000, reward: 17.200000, loss: 0.000690, epsilon: 0.010000, episode:  454\n",
      "frames: 996000, reward: 17.900000, loss: 0.000648, epsilon: 0.010000, episode:  455\n",
      "frames: 997000, reward: 17.900000, loss: 0.000466, epsilon: 0.010000, episode:  455\n",
      "frames: 998000, reward: 17.900000, loss: 0.000777, epsilon: 0.010000, episode:  455\n",
      "frames: 999000, reward: 18.100000, loss: 0.000697, epsilon: 0.010000, episode:  456\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "    \n",
    "# Training DQN in PongNoFrameskip-v4 \n",
    "env = make_atari('PongNoFrameskip-v4')\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True)\n",
    "\n",
    "gamma = 0.99\n",
    "epsilon_max = 1\n",
    "epsilon_min = 0.01\n",
    "eps_decay = 30000\n",
    "frames = 1000000\n",
    "USE_CUDA = True\n",
    "learning_rate = 2e-4\n",
    "max_buff = 100000\n",
    "update_tar_interval = 1000\n",
    "batch_size = 32\n",
    "print_interval = 1000\n",
    "log_interval = 1000\n",
    "learning_start = 10000\n",
    "win_reward = 18     # Pong-v4\n",
    "win_break = True\n",
    "\n",
    "action_space = env.action_space\n",
    "action_dim = env.action_space.n\n",
    "state_dim = env.observation_space.shape[0]\n",
    "state_channel = env.observation_space.shape[2]\n",
    "agent = DQN_PERAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate)\n",
    "\n",
    "frame = env.reset()\n",
    "\n",
    "episode_reward = 0\n",
    "all_rewards = []\n",
    "losses = []\n",
    "episode_num = 0\n",
    "is_win = False\n",
    "# tensorboard\n",
    "summary_writer = SummaryWriter(log_dir = \"DQN_PER\", comment= \"good_makeatari\")\n",
    "\n",
    "# e-greedy decay\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_min + (epsilon_max - epsilon_min) * math.exp(\n",
    "            -1. * frame_idx / eps_decay)\n",
    "# plt.plot([epsilon_by_frame(i) for i in range(10000)])\n",
    "\n",
    "for i in range(frames):\n",
    "    epsilon = epsilon_by_frame(i)\n",
    "    state_tensor = agent.observe(frame)\n",
    "    action = agent.act(state_tensor, epsilon)\n",
    "    \n",
    "    next_frame, reward, done, _ = env.step(action)\n",
    "    \n",
    "    episode_reward += reward\n",
    "    agent.memory_buffer.push(frame, action, reward, next_frame, done)\n",
    "    frame = next_frame\n",
    "    \n",
    "    loss = 0\n",
    "    if agent.memory_buffer.size() >= learning_start:\n",
    "        loss = agent.learn_from_experience(batch_size)\n",
    "        losses.append(loss)\n",
    " \n",
    "    if i % print_interval == 0:\n",
    "        print(\"frames: %5d, reward: %5f, loss: %4f, epsilon: %5f, episode: %4d\" % (i, np.mean(all_rewards[-10:]), loss, epsilon, episode_num))\n",
    "        summary_writer.add_scalar(\"Temporal Difference Loss\", loss, i)\n",
    "        summary_writer.add_scalar(\"Mean Reward\", np.mean(all_rewards[-10:]), i)\n",
    "        summary_writer.add_scalar(\"Epsilon\", epsilon, i)\n",
    "        \n",
    "    if i % update_tar_interval == 0:\n",
    "        agent.DQN_target.load_state_dict(agent.DQN.state_dict())\n",
    "    \n",
    "    if done:\n",
    "        \n",
    "        frame = env.reset()\n",
    "        \n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        episode_num += 1\n",
    "        avg_reward = float(np.mean(all_rewards[-100:]))\n",
    "\n",
    "summary_writer.close()\n",
    "torch.save(agent.DQN.state_dict(), \"trained model/DQN_PER_dict.pth.tar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwYAAAE/CAYAAAD8LNSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydd5wdVfn/P88tW5Ld9JAeQgmB0CHSew2CXxC+FlTk61eNDQWxoWIDvwL6UxBpIiBY6CLSQyihJEBIISG9kbLJZkuy2V5uOb8/Zs7cM3PPlNv37j7v1wv23pk5Z56ZO7n3ec7TSAgBhmEYhmEYhmEGN6FSC8AwDMMwDMMwTOlhw4BhGIZhGIZhGDYMGIZhGIZhGIZhw4BhGIZhGIZhGLBhwDAMwzAMwzAM2DBgGIZhGIZhGAZsGAwoiGgGES0jonYi+k6p5WFKCxHNJ6KvlFoOhmGYcoOIthDROaWWg2GKDRsGA4sfApgvhKgVQtxeamGcENEniGglEXUQ0UIimqnsqySiW4loJxG1ENFdRBRV9h9CRK8RUSsRbSSiTzrm/oq5vYOIXiKiicq+EUT0EBE1mv/90jH2JCJaZBpUK4jolALehn4JER1GRHOJqJmI0pqbENE0InrB/Gx2EdEdRBRxmWsCET1jfpaCiKYVWn6GYRiGYXKHDYOBxb4AVrntJKJwEWVxnns6gH8C+DqAEQCeBfCMolxeB2AWgMMAHATgGADXm2MjAP4D4DkAowDMAfAPIjrI3H86gN8AuNjc/xGAR5TT3wpgCIBpAI4DcAURfckcOwrAMwB+Z8r1WwDPEtHILK6RiKjo/6bcFPQMiQF4HMCXXfbfBaARwAQARwE4HcA3XY5NAngJwGV5kIthGIZhmCLBhsEAgYheA3AmgDvMVfODiOhBIrrbXOntBHAmEV1ohhu1EdF2dfXcXBUWRPQlc18LEX2diD5mrqTvJaI7HOf9XyJaYx47l4j2dRHxfABvCSHeFkLEAdwCYBIMBRMAPgHgdiHEHiFEE4DbAfyvue9gABMB3CqESAghXgOwAMAVytgnhBCrhBB9AG4EcBoRHaDs/60QoksIsQXA/crcJwFoEEI8Yc79DwBNAC4NeN/nE9H/EdECAF0A9iei4UR0PxHVE9EOIvq1NMqIaCsRHWu+/oJ5v2ea779CRE+br48jonfMe15vrtBXKOcVRPQtItoAYIO57VwiWmt6Ve4AQEGuAQCEEOuEEPfD3bDcD8DjQogeIcQuGIr/oS5zNQgh7gLwftDzMwzD9EdMb/Ztpgd0p/m60tw3hoieM7+n9xDRW3JxiIh+ZH7/txPROiI6u7RXwjDBYMNggCCEOAvAWwCuEkLUCCHWm7s+B+D/ANQCeBtAJ4AvwlgdvxDAN4joEsd0xwOYDuAzAG4D8FMA58BQBD9trtDDHPcTGEr0WPP8j0APwa6oyveHeeyfTETDoVdw/cZC2Q/Nfrexzv1BuAKGF6MWwFYADwGIAzgQwNEAzgMgY/3fAHCG+fo0AJuRMo5OM/cDQALAdwGMAXAigLORvkJ/CYzPaiYRjQHwLxheljEANgE42bogoqnmj9fUDK5L5Y8APktEQ4hoEoALYBgHDMMwA5mfAjgBhqf0SBhe5+vNfd8DUAfj928cjN9DQUQzAFwF4GNCiFoYC2Nbiis2w2QHGwYDn/8IIRYIIZLmau98IcSH5vsVMBT50x1jbjSPfRmGIfGIEKJRCLEDhvJ/tHnc1wDcJIRYY3oBfgPgKBevwTwApxPRGebK908AVMAI8QGAFwFcTURjiWg8AJk8PQTAWhhhLD8goigRnWfKLMe+AMNgOYKIqgH8HIBQ9r8E4DoiqiWiA2F4C+S+hQAmEtHl5txXAjhA2R+EB01vRRxGKNMFAK4RQnQKIRphhDJ91jz2DaTu96kAblLen27uhxBiiRDiXSFE3PRy/Bnpn9NNpoelG8DHAawWQjwphIjBMOh2yQOFENuEECOEENsyuC6VN2AYhm0wfggXA3g6y7kYhmHKhc8DuMH8DWwC8CukvNUxGOGV+wohYkKIt4QQAsbCTiWMRZuoEGKLEGJTSaRnmAxhw2Dgs119Q0THE9HrRNRERK0wYv7HOMY0KK+7Ne9rzNf7AvijuRK9F8AeGKvtk5xCCCHWArgSwB0A6s1zroahZAKGV2MZgA9gKOtPw/jSbTQV3UtgeDh2wVileVyOFUK8CuAXMFbMt8JYmWlX5v6OKfcGGLkKjyhjd8PITbjWvM7ZAF5RxgZBvcf7AogCqFfuy58B7GPufwPAqabxEwbwGICTzQTd4eb1wwwFe46MRN82GEaX83NSzztRfW/+OG1HHjBd43MBPAVgqCnHSBjhYAzDMAOZiTB+VyRbzW2AkZu2EcDLRLSZiK4DACHERgDXAPglgEYiepSUghgM059hw2Dg46ww8zCMZNspQojhAO5BBrHoDrYD+Jq5Ei3/qxZCLNQKYqxmHyaEGA1Dkd8XZhy6EKJbCHGVEGKSEGJ/ALsBLBFCJMz9K4QQpwshRgshzgewP4BFytx3CiGmCyH2gWEgRACsNPftEUJ8XggxXghxKIznXh37hhDiY0KIUTBWgmao+wOg3uPtAHoBjFHuyTDzvPIHowuGsfKmEKIdhrEzB8DbQoikOc/dMDwl04UQw2B4WJyfk3reegBT5BsiIvV9jowy57pDCNFrGlN/heGlYBiGGcjshPFbJZlqboMQol0I8T3zN+sTAK6VuQRCiIeFEKeYYwV4IYUpE9gwGHzUAtgjhOghouNg5CBkyz0AfkxEhwKAmXT7KbeDiehYIgoT0VgYq+jPmp4EENEkIppIBicA+BkM40GOPYKIqswY9+/DcN8+aO6rIqPcJpkx9PcC+KMQosXcfwARjTbPfQEMJfzXytxHm2FEwwD8PwB1Qoi52dwQIUQ9gJcB/J6IhhFRyDy/Ggb0Boz4U5lPMN/xHjA+pzYAHUR0MIBv+Jz6eQCHEtGlZFQp+g6A8UHlNu9dFYzwLnlPK81raoZR6ekbRBQhohEwvD/LPearguFKB4BK8z3DMEy58QiA680w1zEwQlX/AQBEdBERHWguxLTBCCFKkNFT6CzzO7QHhsc6USL5GSYj2DAYfHwTwA1E1A7jC+7xbCcSQvwbxirIo2a4y0oY8fVu/BHAXgDrzL9fVfYdACOEqBNG8u51Zo6D5AoYq+KNMBJxzxVC9Jr7qmB4QjpgrPS/A8OwkBwL4EMY4UU3Afi8EEKtvvNDAM0wVvsnALB6JBDRqUTU4Xkj0vkiDAV7NYAWAE+a80regKH4v+nyHgC+D8NoawfwFxghR66YyvunANwMw9syHUblJnkdU8moVuWWfLwvjB8veV+6YXxOkkthhFk1wXCdx2EkR8v5O4joVOX4bhifB2B4Prq95GcYhumn/BpGTtUKGL8jS5FaWJoOI/S0A8bvzl1CiPkwFkVuhvG7sgtGKOlPiio1w2QJGaHIDMMwDMMwDMMMZthjwDAMwzAMwzAMGwYMwzAMwzAMw7BhwDAMwzAMwzAM2DBgGIZhGIZhGAZsGDAMwzAMwzAMA6MJVL9hzJgxYtq0aaUWg2EYpt+xZMmSZiHE2FLLUWr4d4JhGEZPPn4n+pVhMG3aNCxevLjUYjAMw/Q7iGhrqWXoD/DvBMMwjJ58/E5wKBHDMAzDMAzDMGwYMAzDMAzDMAzDhgHDMAzDMAzDMGDDgGEYhmEYhmEYsGHAMAzDMAzDMAzYMGAYhmEYhmEYBmwYMAzDMAzDMAwDNgwYhmEYhmEYhgEbBgzDMAzDMAzDgA0DpoC8s2k3XviwHkKIwGPW7WrHrtYe6/3KHa1o7ui13scTSSzY2Ow7z8KNzYglkgAAIQTeXN+EZFIvR188iYUbm/HG+ibMW90AAPiouRNbd3dqj29q78Xj729HfWs3Vu5oRVN7L/riSTy9bAcee38b9nT22Y7fursTq3e2YdFHe2zb1ze0Y+febgDA7o5erNzRCgBIJgXe2tBk3bdYwpCvN57Awk2pa+/sjWPxltScCzY2I25esxfbdnfhkUXb8NLKXejsjePJJXV4ZNE2vL62Me3YtzY0YdXOVuszaWrvxaqdrdp5l2xtQXtPzFU+APiwrhX/XlaH3njC8zPxYmNjB7bv6bLer6jbi6eX7UBf3PvanffPjZbOPizb1oJnl+/Eo4u2YVdrT9pn8vaGZjyyaBseX7wdrd0xvL9lD7r7EtYcc1ftwtsbmvHih/WBzskwTOFZu6sNDW09/gcyzCAmUmoBmIHL5X95FwDwyrWn48B9agKNOf+2NwEAW26+EABw0Z/exvhhVXj3J2cDAG59ZT3ufH0Tnvj6ifjYtFHaOd7fsgefu+89fOvMA/CD8w/Giyt34Zv/XIobLj4UXzxxWtrxd7y2Abe/ttF6/95PzsaZ/2++TQ6V+97ajD+/uRmfO34qXl7VgI8fPh5nztgH1zz2AQCgoa0X3zl7unX86b+bb71e9rNzMXJoBQDgvFtT13rJXQuwfU83ttx8If72zhb88tnVuPvzx+CCwyfg9y+vxz1vbMKB+9RgY2MHXvve6dh/bA2ufnQZXlnTiA9+fi5W17fh8/e9h2vOmY5rzjnI8x7f8tJaPP9hPQDg22cdiD8p17785+dh+JAoAGD1zjZccf8iAECIgM03XYizfj8f7T3xtPvS3ZfAZXcvxEkHjMbDXz0BAHDNYx9g3uoGLL7+HIypqQQAXPHAe9jbFcNb65vx1LId+PUlh+ELJ+zrKa+Tax5bhgnDq/GXL84CAHzmz++iO5bAqKEVOO2gsa7jbnphLR5cuAUvfOdUzJw4zPW4S+5agK27U4bH5cdNxUHjavCrZ1fjni8cgzNm7IMr/7oICdOo2dzUiXve2IQLDhuPu79wLOpauvC1vy+xzal7jhiGKS6zb3vL+i5jGEYPewyYgtMTS/gf5MEuZYVnfUMHAGB3R5/b4ag3V7e3mMqdXJVXlT2VhrZe2/vO3rinPC1dxrnbumPY09mLbXu6bF6C3rj79bqtj2/fY8iYSApsM1/vMOX+qNm45o2Nxl+5yL50217zfElrBX1HS7en7ABssm5uNrwi155rGBMdfalr36asystztvfo7008mTRlarG2rd7ZBsD++e/tMjwKK02vg7zGTNi2uwstyjV0m/N39Xk/Z+t2tZsyuD87QPpzUtfSZX0+dS3dqG/tQSIpcP2FhwAA2kwvyZKtxrV3a+RIZOEZYRgm//A/RYbxJmfDgIimENHrRLSGiFYR0dXm9lFENI+INph/R+YuLlOOxAKEt+jQKVMy9CQcItdxMqSkMhzs8R5dU2F776dgSuW4pasPSQHsau1Bh2JMeP3wJH3Cqjr74pCXJq8/4riOpBJiBAC9sSQ6ew2Zh1SEPecHgI7euHVc/d5uhEOEfUcPAWBX4ne1BlfaTbsA8USwX115D6Men6OOzt442nriWgMl2+fMizE1ldjV2gP5ESSSAvXmfZk5wfA6yOdNGih9GjkKIRvDMAzD5Jt8eAziAL4nhDgEwAkAvkVEMwFcB+BVIcR0AK+a75lBSDzLJRqdp0EqxV76pFTCogENg1FD7YZBXYvesyCRRkBTu+Fp2Lm329pG5K38+8XUd/TEETIvTh5a4bgOqYhKw6E7lrCU0iGV/tGBHb1xTBheBcAwamoqI6iKGoaCutpd3xo8Fld6DIJ+1h2mYu80evyQMnVovDqFUL6njqpGfWsPwiFDznhSWPkWE0ZUoyIcsu59byxpypF+D9gwYBiGYcqBnA0DIUS9EGKp+bodwBoAkwBcDOAh87CHAFyS67mY8iSoUuQ8Tm8YGH9D5O8xqIhk93hvcQk5ksjVamkYtPXE0dDWg4pICFWRMLycAn56c3tP3Lo2aWBEw/Zrlcq3XJ3vjiXQZYYADYn6ewzae+KYMLwaAFDfZhgG1eY4NQwqE8PAK1RGvR/S09PeKw2DzDwGUimXSc7qMxPUW6Fb0U/NYd+37+ih6OiNW8+i4TEwZBg/rArRMFnGlJxXlwAeVLaBBBE9QESNRLTS57iPEVGCiP67WLIxDMMwevKaY0BE0wAcDeA9AOOEEPWAYTwA2Cef52L6N2olIqk0tnT24fZXN+CPr2ywlGqVbochoL5/dU0D6lq6Uh6DEGHljlYrrlslU4+BU1HUVSNa9NEerN3VhoUbm/HBdiO2v6UrVYFnQ0MHaisjCJHhFXjxw3o0aqpfNLb34AUz8VdFKswdvTHLGzJvdQN27O1OW1WX1xczV+m37enCv5bsAABUmyFCCzY2Y3OTkZOwqakDb21ossa398Qw3vQYCAHUVqkeg9S9qPcJJXp1TYOVIxDzMAyaOnrxonnNYYdBN3dVA+6ev8k1L+P5FfV4fV2jlbuw05SpozcOIYTtGXl9XSM+atZXklpT34bFW40KST2xBJZv34vX1zXi2eU7ARgJ62vq29DoeC6njDQMqMZ247Ns7Y7hjtc2YsSQKKorwoiEQ2k5BRxKZPEggNleBxBRGMAtAOYWQyCGyYT1De2+OWcMM9DIW1UiIqoB8C8A1wgh2shjRdcxbg6AOQAwderUfInDlBh1lViuls5b04A/zFsPABg1NIorHBWCnB6CnlhKmfryQ4sxpqYCM8bXWtsu+tPbANIrvkjFTHoM/OL6Y3Fjf1U0hJ5Y0lYuVfLpP7/jOceGxnbUVEXQF0+ipSuGb/xzKY6ZOgJPffNkjBtWaSU4/++Di9Hc0YvlvzjPGptIClRGQujqS6C9J24ZCR9s34tL71qA2YeOd8gry7Aa77/zyDJrn1TwP3/fewCMe3P279+wXvfFk+iNJ61QIgA2j4H6GTg9Bs6ys19+aDFGD63Akp+di4THiviVDyxCe08cq2843wo5kizfvhfLt+/FoROHpVUUauuJ4VsPL7Xeb7n5QuuzSQrDcFTlfXHlLry4cpe2AtAFf3zLet0TS+LiOxdY70/YfzQ+dY/x+T737VNs4yaMMAwDmTQ9f10jumMJ6zmMhkPocjy3ulAiLy/FQEUI8aa5WOTFt2H8bnys4AIxTAYkkgLn3fomTjlwDP7xleNLLQ7DFI28eAyIKArjy/2fQoinzM0NRDTB3D8BQHqRdABCiHuFELOEELPGjnUvNciUF6pqFLNCLFJbdbHoPX3eoUTNHX1Kkqu7oiUV/QozTEUaGG72QSyRRIiAf33jJAD2+PWgdfabO/pQWxUBUWqlfW+3oUxWKeE9sieDukLe2Re3jJiO3jhUo7qhrTfNY+ClZPr1jJCrX6OGVqDSPKfhMTBeyxX4ZFKk1fvuVfoEyPuy26wO5FT4VWToVVdfwjWUSued0CUYq8e198TTnpkgOD1TKs7nssbM2ZBerzZTpts/ezQA4xnrcXgMYpp+CjpjYbBDRJMAfBLAPQGOnUNEi4locVNTk9/hDJMzckHp3c27SywJwxSXfFQlIgD3A1gjhPiDsusZAFear68E8J9cz8WUD6qCKpUtoZgLOv21J+70GLgnH3spWn0JY5wMJZKKoFs4RyyZRDQcshRlWeFHJ5MXNZURhENklUfdp9ao3R9PiLRkabUZV0dP3Eowbu+J2+5diNLj8L3i1b1i/WOJpGX01FRGUFtlKL01VVHLeJH3vLmzN+0eq4q60zgJUo7TrdQpoM9nUJulAYYxpR7X3hP3VPLdcIb+JG1hb/brkoaBNIqkYSUNqWgkhK6Y/bp0RlKQxnODkNsA/EgI4fsh8gISwzBMcchHKNHJAK4A8CERfWBu+wmAmwE8TkRfBrANwKfycC6mTNB5DFTdURfe41TYnEpfSKn446WISqU7air6Utl166cQiwtEwyFUhA3luFOp5d8TS2JIhXaYRUUkhL54EjWVUYSIsHOvobyOra2yZI2GQ7YVd7UkanuP4jHoiduMhqpoGNGQPsdAh9NmUD0eHT1xq+Z+bVUENZURNHf02aoSyXukC6dSPSlOwyBINSKnoq9Sv1dzPoch0dDai/q9Pdb97uiNI7PUZQOnsafK7jS6ZFlXaRDIz63K3B4JEVq7nTkGHEoUkFkAHjU9ZGMAfJyI4kKIp0srFsMwzOAlZ8NACPE24Pr7fHau8zPliar3SyXeL8wlLfnYYShURELWXF6hK3KlWxoRUtl1W12OJZKIhsmmnLvJpOOAsTVYU9+GYVUREJGlBI42y6DGkwIVEbthoK6ed/TGEFGq9fQ6DAOnx6AvkXRdgU4mhc1o2q00AuvojVvXVlsVRW2V0eF4WFXESlqW17vTR1Hvjfl7DJxpRp4eA02idrsj6W9nazfqW7tx4NgarK5vQ0dPPOOqRkD6c6XmRzivY6jpMXD2tqiKGPcrGg7ZDE4hBIcSBUQIsZ98TUQPAniOjQKGYZjSkrfkY2Zwsr6hHe9u3o3mjj584/QDsKGxHSvqWm0dbZvae3H7qxus0BUgpbS/vGoXaiojaO2OWd1wAeBfS+qwpr7Ndq6KcMjyOvzlrc3W9jtf34iOXqNk6IWHT7AU62eX12NYVRSPLNoOwJ7MLFn00R48s3wnKiMhyzBQPQa3v7IBN192uOv1R0KEySOrsaa+DTVVEVvIkFSME8lkWi+C37ywxnrd3hO3FMeFG5uxYkfqPlRGQmnVleIJgaeW7tDK89ji7VizK3Xfbn91g/V6/vomLDWrOBleglDqteVdMe6RrrnZHa+n5lJXwP/53la8uiaVQnT3/E34xJET0sa7eQwqwiHUazogOz0Gm5o60NYTx/RxhmFw2yvrcdVZB6aN6+qL4575mzCmthLH7zfalrAOAH96baPtvWpkuuUYqN6ScIisErIVkZBVKtY4d8I1lOippXXYf2wNjpoyIm3/QISIHgFwBoAxRFQH4BcAogAghPDNK2AYhmGKDxsGTE6cd+ubqTdC4HaH0gUAN724FgBs1XWk82DO35do5/3eE8vTtlVEwpbXYeWOlPL7u7nrrNcNbT1Wjf419W24/ulUCXVdKJGsNjRpRLWVY6AaEI8t3o5rzp1uG3PYpGE4dupIPPTOVlRHwzj3kHFYt6sdJ+4/Gi+varCOS3k3BIY6OhKrZVa7+hKWor3YUX61Khq2vAmSWCKJJ5bU2bbVVkXQ3hPHxsYObGzssLb//d2t1uufKfdiaGUY5xwyDo3tvTh22khEwiGjJr95j2Qp1js+dzTueWMTVu5ow1zl2tRwp5/+216m/paX1mpLsrZ16z0G08fVYJumd4TTw7Cp0ShFeszUkfjPBzuxeGuL1gvx7ubdtudQV6VIRfUSOD0GQyrtoUQAUBUJWQniUcVYlce5hRJd+/jyQPIMFIQQl2dw7P8UUBSGYRgmIHntY8AMbno1IRQqYSXsI5tmyJWREBI+4Ug9saRNabXvcw8LUkOJnDS2pWrbj6mpwHPfPhXTxxmr0COGRvHpj03Bmz88ExccPsHmMZCrz4mkQDTiHvISS7jLXBkJpTVzk8efOWOstep//5WZVXusiobxtdMPwBs/OBMnHTDG2BYJ2xp5hUOEi46YiOe+fSpuvOQw23g3eSW7O9L7VLS5eAwOGleL9t54mkeho9f+Xo4fN6wKP5p9MABoa4x7hSzpsOUYOB5MWcZV3V6tGHlOo60nluRQIobp58heNAzDpMOGAVM01K68ApkrSpWREDxSCwAY8eNueQFe+QLRcAiREKXFxQP2EpkytEQqjCMdmclqqVEZux43k4/diCeEu2EQDafdqz7z+IpIyJJjktmIKyjVmg7JVRUpwyBuGgaSiUrfA8DbyHLDTWGfPq4GQHrCszOUqNUs/zqsKmIp5B15MAzsHgP756AzFisjYdf93bGENjlcZywwDFMaLlH6mDAMY4cNA6ZoqEq3z8K/lopIyLdZWU884aq0ensMjPAQZy4AYC+lKav3yL8jHIaBWkBI9RjIeZ1lSwHTA+CSTFwZCaV5V+Lm8RWRMKqiYYRDhHFmadSgVOkMA7PBmyFz0rYaPt5hGPh5h3S4Ggb7GN6XnQ7DwJl83GqGN9VURZRO0elz6rbpDCGJl8dA9zyoHgOnwdcTS2grNKl5B0FKuzIMwzBMKWDDgMkfPgViVIXJr0KRDiG8uxhXR8Po6fMyDNyV2YiSTOpENQykIijPMXJI1HasGvaTSCYhhFElSM7rDAsCjFVmV2VRc80ylKgibHgMxtVWIhIOaY0ON3SGQXU0bFXsSfcY2D0Sfh6Dzr5EmvfFLZRo+j7SY2BPQHYaEnJ8TWXE+rycXgVjXPp5hla6GwYJD6VdZyzK8C0gPZSoO5bQen/UvINmTZgVwzAMw/QHOPmYyZq/Lvgoo+PVmvrZLJr2xhM2ZdXJ0MoIeuJJVwPAK5RIKt+VkRDaHft2KhVzpEIqlU9nKJGq+MeSwjKGpEGhC1WSpTCHVoTR2eesiZ9M8678v5fXAzCMmMpoGCNM4yQcIiQDxrLr7mNVNGzdo0RS2JTeEQ4D6NuPLPOcv7U7ZlXukbhVJZo0shpERonUXz6zCvWt3dintgrvbLJ3HJWhRKrH4L63059Bp7HwwNsf2cJ/nKjx/7pcgIpIyObRUb0P0Ui6x0AXSqQaUvWtPRg3rCrtGIZh8sdbG5owrCqKI32qgAkh8OSSOlx81CTXPDOGGUywYcBkza+eXZ3R8arOlYnD4IwZYzF/XRP64klUeoSE1FZFsKu1R6uYzZwwDOsa2q2kWieysZVcHZ4xrhbH7DsC81Y3oKUr1QtANhu77NjJWFHXiqvPtlcsUhX/RCLVU0AqyUYOgv3iZejLV0/bH08srrOVeo0lkjaDSqUyEsIXTpiKoRXGP+OQZm6JzuhwUhEOWffO8BikfiSJCN884wDcNX8TgJSS7mRMTSVGDY1ifUMH9nbZj5EegO+fdxC6+hI45cAxWLRlD6LhEGoqI2jp6sPf3tmqm9Z2zmFVUZvRcuA+NbZKTM4QpBueW40ZZrL4V0/dD0lhGA/vb92DzU2dthV+Z44BYHqRlEV+1dvi9Ca4hRI5eycwDFNYrrh/EQD/KmDPf1iPHzy5Atv2dOF7580ohmgM069h85jJGwTyDGdRlS6/XAHJ6KEVePBLx+Hy46aiz6N6D2CEmPTEE2lzX3bMZHzu+KlIJKINQtsAACAASURBVAWa2vVhHJbHwMohCOGmS4/AYZOG25RgaVTUVkXxh88chZFD3T0GccVjUGGuWOuMFllZZ0xNJf7yxVnW9tmHjkc8IVy9K5WRED5//L645OhJNtl0hALEGYVCZBkyiYRIC5P54eyD8eMLDvac49E5x+ObZxi9BZwKsjQMTjxgDH44+2CcdOAYXHPOQda17FGaseno6ksgEiJURkKIKEaLUyZdLkNSCFxw2Hj89MKZ+NlFM3HLfx+B3/33kQDs+RI6pb7SsYqoPl5Or4hbVSzZG+P6Cw8ZNH0MGKYckGWUvUL8DvvFXNyn9M5hmIEMGwZM3iCCZ8iGGr8d1GEg9exKs3OwW5IuYBgGQuhzCSaOMEI3dmoadwEphVCuAMvQn5rKiG3l2xk64kTVpRPJpFWZSM6rs4c6e43V5IpIyNYELhoxVvDdjCin2zusi1OS+wIYBmEi61yxZFI7xuMU5nlC2vwFIJUj4FSmAeO52d3hbRgARhgREdk6Hkccq/a6kKWEEGnGkTR8euOp1XxdrofzPqseJOe53aoSdZmfsVd1KoZh+icdvXH8+vk1/gcyzACAf6WYvEEAKqPuj5SqLwVPPjaUt8pICH1xb4/BULOUaJcmbEM2PXOWxEzJJlf27YZBbVXEqoYDAFEfBTvdY5A053UfJz0GlZGQVQ5VnsvIMXAxDBxKppfSrkt6dhJWPQZJYVO+g84TCZEtOVdFruTrlOOKSAi7O/2TcqXhpBotzs9E53lIJkWa4STn6FUMybgux8BRUUo1FHWhRFrDoI8NA4ZhGKb/w79STF5xhl2oyFCicIgC5xhIXU4mgHoZBlJp7O5LDyWZYJbbVBOJVawcA1N+qRTXVEZsMet+K++2PgZJNcfA/b7IMJPKSMgybuQYw2OgH5fmMfAKJQpgGIRCZOWBOKsSSchnnnCIXEuDtlseA41hEPYPJQKAmkojCVoNc3LKqZsnIURamJv8jHvj3qVE5X2eOmoIALvHwOn96I4ltMZFdyyuPZ5hmP7F08t2YPZtb3oeE/fwXDNMucOGAZM37pq/CQ1t7qu+Ul8KUfAcA6lGVYRDEMK7spAsSdnlOGZoZRjDq6OojoZR39qDbbu7cM4f3sAFf3wrJZsjlKjC8hjYq/EMr7a/d6Iqn73xJD5510IA3oaBTD6uiIRsyn40QmaOQbBQoqByuREmYPn2vfjkXQvQF0+m5RgEmcfwGOgNA1nxR6ccVwTIMQCA2krpMVBKhjru7W6dYZAIFkqkC1WT93mKaRio99157p6YEe7m9Jp0cigRw+SF9WYRiUx5MEAVvbaeGK557AOs3eWsTZfitbUNOPCnL2LljtaMZWCYcoB/pZiikUwaq7YEcs0xmDSiGledeSDG1BhJvarHwA+5mqzq0ZWREL5//gwQEYZVR9DRE8f9b2/GxsYOrKlvs46TOQbSuFBzDCQjhkRx/YUzPWVQV+Z37u22KgypCuHQCrvi3GCGN8lmaTdfejie/84piIYNL4n8DbznC8fYxjnviddPZaAcA1PZXrZtL/Z09tkSfCV+s4RDZGsApsMtlCjIb70MVVONFqehobOjeuJJTSiRMZfqMejVGJ7SSBxaEcENFx+Kx+acqJw7ZM5FZoM4I5RI9ZpEQsShRAyTBzY2tuO8W9/EH+ats7a9trYBD2jKFqv0xBL4ZYAqekf88mXfY15b2wgAuPnFtWwcMAMS/pViikY8mQQRgTw8BlVRQ5GXCjnBvfEYAFuFlxpNE6sfnD8Dw8xVfxmOpAuHkfLI88ok4xolGfj7583A8CHBPQZqTXw1xOqiIybaxsiOv7KJ2GePm4pDJw63QokEBCojIcw+bIJtnK4rr7tcQQyD1OukcMkx8DEwIqEQqjQJ6KqnxS2UKAjyOlRDR2fAOOnuS++BEdHkGPRoQtWsvJNICF88cRpmjK9V5E7NWR0NG+VKE8Le6yAc4lAihskDjaZHeunWvda2/31wMW54brW243k+0FUqA4C3Nzbjoj+9nfF8/15Wh2nXPe/bJPLG51Zj2nXPZzw/w+QKGwZM0UhIj4F7uX1LyXMq726GgbpdXd1PzZeapyIccs1RkDGj0hCQCa21ypy60Bonqtydyg+VqhBGNYnI4RBhbG2lbVs0bIQSCaFX7J09HbyiswLozjbFOZltjkGYUFWRfrJJI1Kdk3VGQNCwKCmT+lnoDBgn3bFEmlET1oQS9cYSaeFSstKWTqmXRo4QwmgQ15cwQokqVMOAOJSIYQrMf2WhpKtk0lsnF/4wz2hQ2egRdgsA9/t4QRimUPCvFFM0EkkBIkJIKYvpRCp5UgVLlSvVh6eoSmZNVfpqfljZXxEJozee1FbvkTGrMhwpqskxCNQLQDlEzYdQFUKdcjiutjJNEY+GQ0Zlo0R64iyQrmB7VXryKmUqcVZUymeOweSRKcNAp8hLj4rf/HqPQbBVeOc9SOUYKB6DWDLNAyFli2qsK5ljIGB6DOJJxBJJm9ckGg5ZDc7YMGCYwrC5uVO7/aGFWzzHBfhqtOj08UokkgLTrnve95wAcNf8jcFPzDBFhH+lmKKREIbCT3BfnbGSOcn2J6DHIF0htXkMZCiRJlJeuotlZSMrtKgqM4+BW8iOm2EgQ07Gm1WTdGP6Egm9xyCDHIOg5UolsYS+j4HfPG5ViSYphoFbjgGg9/qoyM9ANS6CKtvO69F5DHri6SFHqVAiTdK0Ikel6TGIJYQtzyIaDqGLQ4kYpiT84plVWY3TGQF+uQzSI33Ti/49Dx59f3tWcjFMoWHDgCkaiWQSIctjoD9GhvCkPAZmjoGL8qduH6pRKlUlz+iFkNCuEMkVbWkYyFhzVVENksDrpjjbqg0pyuEQU4GUfRZU5HH/eHebVuaMqhIpsk/QGCGA0zAQ2SUfEyEaDqXdqykjh1iv3aoSAelVoMbU2MOr5LxqVaJwiKz76IXzHsrrs+UYxBJpBqD0gOi8VqlQIiM/pjeeQNxRlShEwPY9ZhJ6Bp8ZwzCl46gb5qVty6IY0qClvSfmm0fB9E+8l+cYJo8kkmaoCAHCZX3bCiVyaHGySpETVdHShbBEHIZBZ288TbkdXh3Fw189AUDKEJDVaVQFL4hh4LagXuHiMZAyjxyaHgalHicV+2evOgWfuMOIpU0zDMxbetWZB+KIycPR3NGHsbWVmL5PDeb8fbF12L++cZJWRjXUJhOPQW1VxGpeJuWsjoZtyYCHTRqOn378EIwaWqHNU6hUPAYvXn0q6lu70dzRh9mHjccrqxvw25fWYVdbjzW/M8dg7jWn4YUP63HTi2u11+a8PsDIhwDsCcc9sSTCYcI/vny85cX535OnYWxNBT55zOS0OUcNTT2XEbNBXG88ibHR1FfrF0+ahptNuXThSAzDMAONw3/5Mg4eX4uXrjmt1KIwGcK/UkzecVvdTyo5Bq6hRA7FSepyulAb57l0ISWqclsRDmlzDC48YgLGDTPmlyvWMu5cVb4zjdNXibqEvsiQE5nbYB+jrjob4w+fPBxTRlVb16Nj6ughOO/Q8fjc8VNx7sxxmDZmqDX+/EPHYeKIdO8EkB5KpAudUi/viMnDAQAXHj4h7ThnHf9ImPDV0/bHZcemK9fqtdRURXDIhGE46+Bx+PSsKRhWFcWlx0y2cg/kbbR3Pg5hyqgh+NzxU7Vz664PSBkX6qqW9BicMn0MDtynBgAwfVwtrj1vBvYbMzRtTqenRwijF4LaAfzio1JVqHThSAzDlB5d/xMnt76yvgiSDBy8+kEw/Rc2DJi847YoapQrNZRLt0RZt+Rjqbg7URV3naIctSUfG1WJnCvWqsIv+xhIZbEynPJCBAolcrn2ioi9So1EKqe1Vd4VldRTSyXfrY+BTqFPheC4X0PIEUrk5zGQ59ElZTu9N365CalQIr0TM+Q4l+0ehuU99Ckl69rgzF6uNMjnLJkwIvVcGv05BHpjSVQqz50qFycfM0zuuHmcc2He6oasxj25pC7PkjBMaeFfKSbvuNWVT5plN2WOgc44kEq6U490U6gqXWL3rfk0ycdex0iFtkfjMQhSFjNTj4Ez6VlFDcVRjRl5j9IbexlzeSn0XuVGbaFE8aRLH4PUa6u5l2ZOZwKyn7fFL/lYyh+xDBzlcwkYnpPe4Mw0DGL2cqVB5wOAUUOUEDfSewyGqD0NOJSIYbKnHzrcHtckEffEkrjs7oXYvqcr5/m9qs0xTCHgXykmK7y+rNwWXI1ypWZVIghtOJHU3WTlIF0FIRV7Uq+mnKSmj4FTR1WVeWloSGVRnT9IZR83xdtNzlSZ1HSFuLkj5dpWZ5Ur385EOPlWaxhIhdrLMFDG9SWSNuU7JUe6gaM7X5rHwOebRib2unkMnB6PTPsYGDLYjyMihEPkyDFIBJ7POafxXBuVSSo0+SEAhxIxTD7oiSXx/Ir6kpzb7zdJsmRrS78tSSqEYIODcYUNAyYrvKozuNX7NxqcEcj0GLj1MgAUA8HnO9jeOMwnx0CGEjm+2FV7QiaTTh9XmzY+yEqym1Fkz4UgHGl2bI4n06sfSUYria2qUXLoxGEA0lfl5e306j/gFSWjXqtbHwP185AKtM5gcuYY+IXnuFUlcp7Xr4+B2mHZic4oSiQFlm9PdVHdsrsrUC6Jl4x98aRrxSgOJWKY3Plg+1586+Gl2s7AK3e0pim9r6xxDxMajKb6fj9+AV98YFGpxWD6KfwrxWRFwsMyUBWr755zEH7/qSOtMbLzsRBAQmsYkOu7hdedhRevPhVv/+hMS5GuCOtj9yWqMi8NAyf2Up7VeGzOCbjlssM1x2nEdR4ToI9BJBTC3750HJ765kmWPDWalfIvnTwN+44eYs6b2n7zpUfg0TknYMqoIWlj3GSQn4mX10NVthMunY91OQY6XddpRPmGEoW9Q4nCDoNANQZUL828a0/Ds1edop0jaOpAr0t3bDfe+MEZePV7pxtvzFAiV8OAQ4kYpmC8trYBF/3pbTzmCO+56uFlJZIoPyzZugebmzryOudbG5rzOh8zcOBfKSYrvFb7VUV70shqzBhvrL4bhgAhRNKVmT7WqT+qSt/EEdU4ZMIwTB45BNPHGRVj1NAMndLl9Bj0JpJpsjtXxo/ffzSGVOiSgYN4DFwMA0euwvAhURwzdSR6zBr6tZqqRJFwCGccNBaA/T5UV4Rxwv6j046XCXna3ABpGHglH7t0BnY7Rt4PXciRUwa/rtF+OQbyc9R5DFT2qa3C4Wa1JCdBOlcD9o7VQdh39FAcMLYGBEI8mUQiKWwGqwqHEjFM4fio2YjpX9dQvGo4i7bsweOLDUMkS2ejL5fd/Q7O+v0bhZmcYRywYcBkhVd4oro6TEgpc5bHAEa5Up1xIUdKRdjve1Y9V1jnMVA700bC6IsnEXMkIAcNHQkSBeKme0aVHapSK7vu+lfjCSSiKYN70rB3KJHzvXcokTR2dPfFKUPg5GOX+0BpHoPMv7qCVhvq7suuKQ+RvsytymAKJSKiB4iokYhWuuz/PBGtMP9bSERHFltGZvCyc293VuP+/u7WtG0/fHJFruIMWD51z8JSi8BkyOD5lWLyij4MyEDVv2R5UiCVYxAiwyjwCkeitBd6VGVPG0LjaHAGIK0bY9CVZN3KuBPXUB01Nt9mGLiHEgHBQoAkqRwDd8+JZyhRII9B6rU0dnRKv3Osn/zysxnmmnxsn1dnBPoR1ADM1GMgCWIY6O7pAOZBALM99n8E4HQhxBEAbgRwbzGEYhgAOOnm10py3qDJy9lw31ub8TWlmWV/4P0tLdwBucxgw4DJiqChREQppTApqxIRQUCfwOxMOvb7ClVDbHQ6l7MqEQB09tq/pIIqjEGUOjfbIak4KVQDQ97GoCE0XnhWJQoSSuTYp61KpHpoPPoYOGXwr0okQ4n0ycNO+bNRsIMagLkgc0YqXQwDr3KxAw0hxJsA9njsXyiEaDHfvgtA3/2OGbQIIXDLS2szjq//64IthREoz7R09qGrL+5/YEB+/fwazF3VgE15zkfIld++tK7UIjAZwIYBkxVJr+RjWwlHshT2uFWVyDAsdHNYZUoD6k/qwrGfx0Cu4jq/iIMqjLmUKx0xJKXwqkrtKQeOAeCvSAa6H+bt9DQMPOZxKtu6ZO6QzRBLeQyc8jll8AvjGTW0EgAwblildr96riDzTdR0ys7CyZARBLJCw+SzJv8eZVahYlz5MoAXSy0E07+oa+nG3fM34UsPvg+gsKvthcCvIujRN87DrF+/goff25bX8qFn97N8hOaO3lKLwGSAfpmSYXzwKldqyzFQQ4mESL33K1dqGQj6HwK51emdcOKsSgRoPAYBDYNMGpyFQ4REUuC4aaNw82WHW2VQnef7yxdnobmj1/U6w1ZuQBCPgVeDM3M+j3nSPQbeoURWCdEQYdnPzrU1j0szDHzk/9i0kZj33dOsMrFu5w3SjwEAXvruaejsjaOzN46P3/42+gJ0NL7p0sPx46c+9DzGCzWUqDISwqKfnG31Z3j4q8djT2ef1/BBCxGdCcMw0JeTMo6ZA2AOAEydOrVIkjH9hXhi4Nbc7+pL4Cf//hCHTRqGEdUVWLWzFefMHGft53YDTLFhw4DJiqChREBKuZehREbnY+FpXFhjffarCrNuNcnmMQjrPQZBw1KCKOdyqmjYMAzGD6/C/mNr0Nod056vuiLsWnYUUHMMAolojPEI7Qna+dgpp4Q0hkEkRBihdgDWyODnlSEiV6NA7ldl9JtvWFUUw8yeCJVWYzt/4yRXes0qUxXhEPYZlvJaDKmIaCtdDXaI6AgA9wG4QAix2+04IcS9MHMQZs2axaoSkzfyFd23t6sv41LHKr3xJE773esAgCtP3Ncq6x2EhRubcdsrG7I+t465q3bh+P1GpX23MwMf/qVissIzlMjmMSBLobZCiQAzx0ATSuTMMfD50rYnH6fvt+UYWKFEjuTjfOYYyGs1V7jG1lamyRbUQwFkFhfv3eDMP1chPfzHO8dAmm1ehoj1PsdfX+lmz6aoT0UkBPT63/fh1bn/AEqviVvyMZOCiKYCeArAFUKI9aWWh+n/bPSJnS9lN9+jbpjne8zKHa3YtqfL97il21JNF19atcv3+O8+/gEa2vITrvPvZXWoioTxjX8uxYn7j8Yjc07Iy7xM+cCGAZMVQTsfE+yr/tIwcOt87Ew6dospJc1Kuk7pVcN/3AyDoIp6kOOkCHHzBu1jGga2Tr0ZaLdSoQ7iXZFoy5VaIU7Bx3n1Q/Db5jQEck38TSVWZ65wy8/dzzipjOamzBORVWmLDQOAiB4BcAaAMURUB+AXAKIAIIS4B8DPAYwGcJf57zkuhJhVGmmZcuBnT2sr31o89M6WosiRLfNWu3dgduOb/1zqub+ls09rFJz229dx0RET8MPZB2d0vu8+ttx6HcSIYQYe/OvFZIVXuVJV+VSrEgGmwk/Gyo6uXKllCARMurXNHTDHoKM3u+TjQIaBw5BJeQwUw6BQHgM5v0ahD1SuNMMcA6/j0hqc5eiuF1ZideZj5efudy9z9WqooysGUb8CN4QQlwshJgghokKIyUKI+4UQ95hGAYQQXxFCjBRCHGX+x0YBkxPb92TXm6BcEULg6Bv1nopte7pw1/xNadtf+LBee3xDWw82Nuobw83522I8uaQusFzOZONnlu8sqTeHyQz2GDBZETiUCJQWl05kNDjz+p4IqqLZDQPvkJZKU1lzNrDKZ7lSOdWE4VWob+3B0VNH2rY7ZfJDXl+QL9VUuI17boBnjoFjnF/nY2tbkFCiPJUKDRr2pSKVdJ2uLpPEgVTzsUMmDMtKNlW0yqi+8zHDMP60dsWwdFsLDtynxtrW2NaT9/Nk28ywv/Do+9szHuPmgTj+N6+6jnl5dQNeXt2A/z7WXlE4mRRICJHWuPGTdy1Im2NNfTtmTszuu5UpLmwYMFnhpac6KwWpCpOl6ELoE5gD5hZI/BROVbmVcvU5Ox8HXNzNZPX+8uOm4n9OnmYlwNo8BhnUzZSyBVlrscJtPAwkr2sNoszbtxhn1HoRnKFEecrwy8bAkCFCOhlW/ep8CGF4wCoiIay5YXbWRgx7DBgmP3z9H0vwzubd+M+3Tra2HeehuGbLIT9/Cbdcdnje582VD3e0BjrObfW/WHzlb4vx2tpGbLn5Qtt2nefGq2AJ07/gXy8mK7w7H9tDh5zhPiEiJJNwCSVyvPdRKH2TkxUl3E05DRq3HsRjkAp5IcsocJ47o1AiOS7Ad6p6brd5cu18rH4e8nz6alAhx/t85Rjk4jFIH1sVDaO6Imw1mKuuCOclP4BzDBgmez5q7gSQvohTbmSiC/dXvdkZeqvy2trGIkoysPjxUysw7brnSy2GFvYYMFnhZf07+xjommIZHoP0sVZDL/ne5Rxye0YeA5dDC+ExcCrg9qpEGSQfh+T9Ck62hkGQzse6W6Cb0nlPc43fl7+aWYUSRdw9BvlENZrcGtYxDDN4WburrdQiZMzn73uv1CIMSB5ZlHkYWLHgXy8mK7xi3u36pDPHgEBESArvOYJ2/PVTOG1dmF2O9VMY1Xr9QXEeSll6DCzDIIPlJK/+A97Jx/7zBE/Utk+Wr6pEmdw7SYXZZKzghoHtnPzVyjB+xBNJtJS48V8xuynPXZV5VaJSs3z7Xu12TiYeuPCvF5MVXh5euzLuaIpl/ifjup04PQVuulyQZFrAXpXI3WMQzOuQSffhTKr/eJGNMuulhHudOkj4T1BpChVin42B4RVKlE/Uj4pzDBjGn189uxpH3zgvremkk06f/QOBvhyaoxWKHXvdqzw9/cGOIkrCFBP+9WKywrPzsUeOQShk/CeEQFLzPfjJYyYZ46Ti76KKqvH0MycMw8Hj7V1zh1TIVWK9XCp+Xoerz54OAGmVF7zwUmCzyTHIZG3Ga34vuZz3QV/dKDuPQb5QZRxTU4HTDxrrO0aG9RRTV4+yx4BhXFnf0I4XP6y3kmefWroDD7+3zfX4vV0x130Dhd/PW+d7zKOLtqHTI+bfj3yu8te35r9KFNM/4BwDJit0icMSZ/iOM8dACH3n42evOgWHTx5ujEOwUKIQAS9cfWra9mvPPQhfOXV/x7GpyS4+aiJ6YgnMXdXge46rzpqOq86a7n2QRKTkciOTlWurKlEG3+de82cSSqQzItRNXjJlE/LjhS6xevH15wYaW6wcA9WfUrzgBIYpP8679U0AwOihRrfx683GZZ87fmrJZCo1DQEU7eue+hC/eGYV1v36gqzOMTdAF+WgbFean/3mhTX4yccPyWj85qYO7D+2xv9ApujwshaTFV5KYdjhMVCVJMODYBgFTsPAtshMtj9pWDHzGSigql4YCYUs46MQoZJeSmgkg9V0NVk7KF6GQSadj3WeFN116c5WqLCdbEKJUh6D4oUSFdwGYRim3/PY4u1YGbD0aFB640mbUp4Ju/OYz6Emz9775mbf49Xf2XmrG3DW798oeblVRg8bBkxWeIYSOXIM0pqQmV4Dp9MhG8XNqbx6Jdk6S4ZK/bwQKVSeITsZ9TEwjtWFXfmNUZFbMsl90F2COt7LWMm3Ei7PlV3ysWkYFDH5uJgJjQzD9F/+6463sW5Xu28YTya/Q7ESlnHt7kvgl8+symmOtfVGdabVO8uvStNggA0DJiu8+hg4y5WS8pSFKOUxcIYjOT0N1gQapOLlpuhqlVpFjkiYCuIxkFN56a/ZVCXKBC8F2CtHwGnM6Iwb3XBtudI8K+HCCtHKPvk4aH5EtrDHgGFyY9p1z6N1gOUTJAVw/m1v4sGFWzyP8wrP7U88sOAj32vR8e9lnKzshhACTy2tQ0+sf3TiZsOAyRghBP66YIvrflUBJ8faKVHqvXMFxelpMMZ74xaVo1MCVaUyGg5Zk2cSphMUL8U4qwZnmZzbM8HYfZxTLt015KsjdaZ4NW/zo6JIoUQMw+TO9pZUmMxASjr+1bOrPfc3tvcGnmtTUydunbc+o0WtNfVtOa/0A0AySwPmgQUf5Xzugcr8dU249vHl+H9z/RPQiwEnHzMZM399E55dvtN1v02ZdYQShQgQIDPHwD7O7jEImnzsCCWCsWqvD4Oxj0sZKN7nyIZ8lysNUk3ipksPx59e3ZCRV8Aml7Mpm8boCmqoFEoJz6ayUEWRqhJx+BDD5M6TS+qwq81IxP3q3xaXWJr+STb35coHFiGWKA+vRDH42ztbMHlkNc46eFypRUFbj2EAZ2IcFhI2DJiM6er1dnfZqhJBk2MAYfQxcFgG9uNS471wT07WrXanthVKSZQKvJf+nElISziDPIjLj5uKy4/zruqRUSiRj9fFMwG9UMnH/brzsf41wzDBySZMJSf432re6OiN44nF/bejr8rP/2N4T7bcfGGJJel/5EU9IqIHiKiRiFYq20YR0Twi2mD+HZmPczGlxyvxGHDmGDg6H5vbkkJoQomU4yg13gs3SXTD7P0UyJq7EGso+VKMUx6DvEznGeKUnnysMwzSx+lWyvOffJz9vKVocMbeA4Zh+jv57l58w7OrfEOmyoVkUhQl5v+Gfni/8rVu+iCA2Y5t1wF4VQgxHcCr5ntmAOBnGIQcHgNVYZIhPLrOxxl12s0wxMjYli4HkN8vRzlTvlaM5T3JNQ8iVa3J/Zi0cqUBk4915D/HQGQ9b2VUNrsrnrLOHgOGYfoL/1pShw5NY7S/v7sVz63Yif/kqYtxa/fAyQm58fnVOPhnLxW8AlR/zL3ISyiREOJNIprm2HwxgDPM1w8BmA/gR/k4H9O/UXU3Z7nSUAhAkpBIBssx8MNNp/crtRl2eDLyTb5WjKWRlS/bJZNr1vYssJUrdSffDc4k2VQ7qiyWx4AbnDFMRvQHA3pPHmv790dW1O3F955Yrt335vomvLKmMfBcjW2Dp9vxY+8bIVF98aRRrCRHemIJdPUlMMps6tefKWQ63jghRD0AN9OKOAAAIABJREFUmH/3KeC5mCLip6SGlZggZ1WikKmQL9qyBy+ttHdh1FYlcvnh8M898F7tDhEKmnycL8J5DnfK5Fr9jCsLzabCJR/33xwD9T4UujQqwzD54eYX15ZahILS3Ze/cJjjfvNq3uYCjOiDXz6zCt9+ZJnNc79kawumXfc83tu8O2/nenJJHepa0hvD/eHldZ5RAz9+6kOs3ZV7v4VP//kdHHPjvJznKQYlL1dKRHOIaDERLW5qaiq1OEwA/MJawo5cAadCJt8/smibfZy2q65ewXLPLXCvZuRMgv7eeTNwwv6jcPYh+bNZ821k5DvHIBN0Sri66eqzp+OE/Udh9mHjA43NB9l0Pj5235E4b+Y4TB01pAASpSCX1wzDMAOFjU0d2u1zV+3C3FUNGc31+OI6PLhwC55dvhPdSjz/wo3NAIC3NjRnL6hCXzyJ7z+xHJ+65520fbe/thErd7gr/s8s35mX6lgr6vLbAbuQFNIwaCCiCQBg/tX6q4QQ9wohZgkhZo0dO7aA4jD5wi/kztmoLC3HwEVrSq9ehKw1LL+KOuEQYcqoIXh0zomorYpmdxIP8rVgnHK+FN8y0FZ2Uj4Qef+Gae5foRqcZTPvlFFDcO8XZ6G6IpxXmRiGYQYb//lAX6r8Ny+syXiu5o7iluds7uhFIinw/Ip623a/vMnBRiENg2cAXGm+vhLAfwp4LqaI+CXj2BRKsr8PkXuYhb0xmv1v2jl8ZNR7DPSv80m+I0jCefIYWF2ePY8KkAxequRjZJ98XCzIZtiWUBCGYZhBRk8sgR88sdzX2Lj/7c341sNLCyrLPW9swl/e3Oy63y106ZnlO/NeKSob8lWu9BEA7wCYQUR1RPRlADcDOJeINgA413zPDAB6fUp42fsYGK/lJrXzsdc4a7yPguUMa5KH+yXOFioGPN//plNViQqPU3bd6nzQUJ7+1Pm4WNhCidgyYBimH+AM2S1X6lu7PasnPbN8J55YUoebXvDOGWlo8zYcdnf04kdPrkBvPPtqRDe/uBb/5+FBaddUiJKs2pl7PkOu5Ksq0eUuu87Ox/xM/6LPx2MQ1iQRh8zeBSECRKBQIvOvixnhp3f5eQz6s4KpkqpKVIpQovRtQW9bf2pwViz6sWgMU1JaOvtwzxubEA2HcPU500stzqDiaZfQHwDYujs9Gbe/cvm972LL7i6cf+h4VEXdw0Jz/R6+5aW1eHxxXdr2215Zj4ff24ZFPz0np/m7+xK4/ZUNrvv7Q1gTdz5mPFm9sw13zt+IP37mKETMrOI+H0vapuDLv6rHIIhh4COXLB/mZjjoFEhVWS2UXZDvf9L5rkrkhXOVW598XCKPgfm3PyvfzmedYRiDn/1nJZ4z47onjKiytjd3DOxSof2dDY36ROL+yC6zVGqmenNLZ59VmS4IbvPf5qHMZ8Kdr2/EfW/3v94FKiWvSsT0b779yFI8v6IeHzV3Wtv8DAN7VSKy/0W6Un7354/Bd885yPaP16u6EAD87r+PxFdO2Q/H7TdKu19bVdOW61Ae2lsxPRtHTxmBC5QKQ9pwrIBzFer+lsOnVg4yMkwxUUtm+v1+MOVJvrwP1z72AX4/bz2A3Bt7AsDRN87D1/+xJOd5/OjojWP7Hvs9eH/LnrTjuovQTTlX2DBgPOmJGV/ilZGU667XJ5RIFxIUInWfXXU6ZMKwNPey3+rr+OFVuP6ima6Kc5DOx+WAFDPn5OMAlxsKEX564SHWe11Pl6Cx82Vye/OK0wge7BDRA0TUSEQrXfYTEd1ORBuJaAURHVNsGRmG6V88tSy3Lsy6vg35Knsqaensw/x19kKbn/nzOzj1t6/bti3Z2pI29skl6WFK/Q02DBhPpHWrWu5ZhRLJJORQusdAnyhsH5cpfs25ChZKlOf4wHCRcwxCPl6VoPctX52fJf2hUoMffpW0BiEPApjtsf8CANPN/+YAuLsIMjEMM4B5/sN6132xhMDW3Z2u+4PyP39dhP/56/vo6kslEQdNGm7tjqVtiyf61+8bGwaMJ9L6jieDGwYRbfKx+R7pOQYhj6cw+8VX7/j4Qofo5GvVuNidj/0Ng2DXle/bWw45BmwR2BFCvAkg3Zee4mIAfxMG7wIYIXvfMOXNU0vrsKVZr4DVtXQXWRpmMLC5KZjC/8oabUutjNhkniuR9P5BdYYWufHiSndjphSwYcB4Ij0GqkXrV8bLXtJSlitN5Qy4dUK2k52W5TREdPuM1wUqV5rn+ax7WaQFBb/KTUENg7zfX+v6+7/23a+Nl/7FJADblfd15jamzLn28eW48Pa3tPvu7+eJl0zhaO2Oob7V3TDc0+mfjC4g0NZjX3XviydxzxubcpYvCG9taEKHR7lRlX++tw1LtrZg5Q7vrseqgfH3d7bmJF8+4KpEZcTmpg584k9v46VrTsOUUUOKeu54MomrH12G0UMrfT0GdgVcvjD+hIiQJLuW6xlKlKWW5bfaXS45BsWsSgTApnfnYq/l22MwamgFNjd3Ihruv5+bDJ/KdxjVAEZ3o7SPOhHNgRFuhKlTpxZSJiZPdGpivZnBzem/ex17u9JDaSTH3DjPd46HFm7FLS+txWXHTMbFR03E9HE1GFFdkU8xPfnTqxvTti3blp5LILns7oUZzf/Ekjr87lNHZixXPmHDoIx4YkkdOvsSeGb5TnzrzAOLeu5EUlit0GcfOt7zWFUxkq9Uj4FT2deFEuUar+3fxyDLiYOeX7PtH18+PmPFNl85BkHtIL9wq8A5Bso8T3z9xGCDPLjrC8dg7spd2Hf00JznKhROI5jxpQ7AFOX9ZADaoutCiHsB3AsAs2bN6l8BuQzDBMLLKPjhkys8x0q94pU1DQCAfy2tw7+WGom8T+bhN8aLWFz/lfPG+iY8tHAL3t/ibhiUI2wYlBGl1DdiSiiRX4MzXciOWpWIENxjkCnGl4fw9RgUrHKMh8pyyvQxGU9nVSXKUpxMsRkGOeQYyMOOnDICH5umLymbCfvUVuGKE6flPE8hYbsgY54BcBURPQrgeACtQoj+FWzL5ERXXxxDKiJ4dW3ucd3MwEb2uVBR18O8yny65RfoEn2zQfZQcHL1ox/45hkEob9VsuMcAyYQiQySj9MzDLxzDHQKqBWWka2B4JNjoDtnPsnX9CmPQX7m86sLHdIYdfb9QZOPpUUz+BZ3+9l3fMkgokcAvANgBhHVEdGXiejrRPR185AXAGwGsBHAXwB8s0SiMgXi2eXuXXcZJigLN2ZWbnTdrnYIIfDZe98tkEQG+TAKAOC1fmY4s8egDClF6caY4iXIpEFNKlfA/AsCZZJjkJmYynjdnEqOQZmYxKkcg+J85moYmC6UKKjSKw/L0/dmWZBrid2BhhDicp/9AsC3iiQOwzBlSkO7fsXejfNvexM3Xnwo1tQHKyEKAC+trEe00DHGZQIbBowrSUWra1eqAPg1OFNJrfynQoqchgB5livNNvnYb3+hqhLlVxMO5c1jEDAESPks/Co7eREqskHTH2CDgGEYpn+wckdwowAAvv6PpQCATx07uRDilBVsHpURueiyx//mFcz52+KMxnQqzTvUpKHeDFp66zofOy8jqlm+z9Vj4Kf4F8owmDi8GgDyViUhX1WJJgyvAgCMHOItl61yUw7lSodWGp2yp4wsbvWsUuL0jjEMwzADjEHw/c4eg0FCQ1svXl7dkNEYtXdBi2IYdDnK0O0/dqhvcxFbzoD5D2vSiGr86IKDUV0R9j4+C/yGFcow+MHsGThm35E4+cDReZlPp5xnwzfPOAD7jx2KCw7zrihlq9yUQ47B/mNrcO8Vx+KkAzNPuC5XcjVmGWYgsqs1szAQhpH0xZNoau+1bevU9BBY6lEutNg8vni7/0H9HPYYlCHFSjFIKCfa251qPNLuaC4ya9+RrnOkdT5WFMtPz5qC/zpyosvAtBfB0JxHR6FCCSsjYXz88Al5qzJgKeo5fuaRcAgXHTHRVy7/zsfBz3neoeNRUzn41h76W4UJhiklJ9z0aqlFYMqU+97+CB/7v1ds29buak877tH3c1PGL75zgfX6iSV1Oc11/dMrcxrfH2DDoIwodgyzmmOwtzNlDDi7/jnlEpp9ao6BtDe8lPNcr9RPgS0X5c2qSlSCWH1tf4kyuW+lgZT/Mwzz/Ie7Si0Cw+SVzU0dpRah4LBhwLiiegxaulIeA7WnAeAd7pNWlYhSVZW8wmSkApp9udLS5BjkGyuJt0h2QTl2h+4vcIMzhrGzemdrqUVgBimFWkxr7ujzP6jMYcOgDHE+7kIIdPWlx93lilqjd69Ho5AghkHI8hiQVcIySC+BTHWsVN8E7+PKpSqZ9Bgki2QZ+OUYMO6wXcAMRhraevDe5t3afYNBiWL6J1t2d5XkvJmUc++vlIl6xADuCvjDi7Zh5s/nYvue/P5DSCrPd6tHK3MvZCiRWpVIqri6OvmpcebfAnkMyiUkRirnx+2Xe/fgIPhVJWL8KZdni2HywYW3v4XPFLiRFMNkyqKP9pRahLJl8GUGDkBeWmnEcW5u7sSUUfkrD6muUjvbkd/2maMwb00DnvdpY54KIUopS3JeT8Mgx2ZRfrpZuYTJhEKEF68+Na+fqxdlclv6JXzvmMEIewUYZmDBHoMypBRViZzusXHDqnDwuFrfOZwr/yEiK8egkB4DP8W/nMJkDpkwrGjVfXi1O3tyLbHLMAzDMKWGDYMywk/fEHm2GNSqRH2Obse1VZFAoSbpOQapECUv5T3X5GP/zsfZzcswbnAfA4ZhGKbcYcNgAFCoVV4vj0FtVdAVbHsJR6JUtYBAHoNsQ4l8xnH8PJNvUs84P1sMwzBMecKGQTkhS1fmUIarsb0nrUGZG2pVIqdhUFMZcV3NV+XLtSpRpjjLo7pRLjkGTPnBTxbDMAxTrrBhUEZYTXBd7IIg5sJx//cqzv3Dm4HOp1YlcoYSDa2MuK7K25KP5V8lCVnu91y1z7H2o5/izw6D3PDqdj1YyTX8jWEYhmFKDVclGgBkqofsausJdJxX7fxoOBRIuU4pSymlSeZCRDxDiXLrIqvr2mvfz9pbtiy5/hwMLVIydHnCzxYzOJi7ijsbM4Un25BiJjv4170MSWtwVqDzJFwMAyIjPyDIyqiz4ViIUgaHd+dj+bdAOQa8rJs1o2sqSy1Cv4QfKWawMX9dY6lFYBgmz3AoURlRbMVDrUqkIlf6g1jx2eYYZBtJ5Gyo5gY7DJhCwQYCMxhpbO/Buy4dkBmGKR/YYzAAKJQeknAxDGQ1IakAeVVJDTnirglQOh+7jwuaROxEJj77eRrYY8Dkm1zD3ximnLnkjgXY2dqDp791cqlFYRgmB9hjUI4UqcOZWyhRxAzgzyTMRy3lKHMMgijn2SpZXJWIKTbZGrMMMxDY2Wrkrl1y54ISS8IwTC6wYVBG+IbuuFUrytKQSCb12y2PgZSLgAsPn6A9lsj+wgglCtLHILfcAt/OxxxLxOSZXHtvMAzDMOl09SVKLcKggg2DAYDfCqVLRJAv7h4Dexy/EMAdnzsaXzt9f41s5HifMjg8DYMck485x4ApFewxYBiGyR8/+feHpRZhUMGGQRmSqZ7vlivgh1u50pCVY5DSgIhIm0zs3BIK2vlYyUnIBr9VW+5Oy+SbXJ9ZhmEYhik1bBiUEX66rFtHZK9+BF64VSUKWav5LnIo53MeQ4E7H2enZQWN8+ZQIibfsLHJMAzDlDtsGAwA/NSRbHOV3TwNzuorXsq4c+WelLJEgfoYZLn+6tfAjO0CJt+oCfYMwzAMU46wYVCGZKrou+UK+OHmaXDG/3tN79SRgicf68cHxW8YVyVi8g4/UgzDMEyZw4ZBGZGtZyDbUKKEmSRcEbE/Jm5K+8fNykTnzhyfdmxvzKgqUBkJpTofZ9A5OSipTss+HgN2GTAFgm1OhmEYplxhw2AQoMsVCFLCVHoaKh2dyKSnwBnmc+jE4dhy84WYMb5WOdj409Bm1LieMLzKyjHwCrnItSY8VyViio0VYsfPlgURzSaidUS0kYiu0+yfSkSvE9EyIlpBRB8vhZwMwzCMARsGAwg3hUSXKhBL+BsG0qCojNofk1AGT41Ullq6YgCA8cOrrRRp79TjHLUrv+Rj1t6YPJNrXsxAg4jCAO4EcAGAmQAuJ6KZjsOuB/C4EOJoAJ8FcFdxpWQYhulf3PfW5pKenw2DMsSt+lAmoUR9CZfuZZpxFU6PQQYro85jJg6vCtT5OOfkYx/hOEGUyTe55sUMQI4DsFEIsVkI0QfgUQAXO44RAIaZr4cD2FlE+RiGYfodv35+TUnPz4bBAMAvCVgXStQX9zcMZFWiqDPHIAPFx3no6JpKS06veXJVsnxzDFh5YwoEP1oWkwBsV97XmdtUfgngC0RUB+AFAN/WTUREc4hoMREtbmpqKoSsTECEELjlpbXYubcb/LQzzMCDDYMywq9PmVuSsW5cbzzVYnz59r14drmxUPfP97bio+ZO23yVLsnHQXCuzIdDpCQfe3kMcovX9hvGfQyYfMOegjR0d8T5bXQ5gAeFEJMBfBzA34ko7XdJCHGvEGKWEGLW2LFjCyAqE5SVO9pw9/xNuOrhpaUWhWGYAhAptQBMcKRC7R4y5D1ORfUYXHznAgDARUdMwE//vRLDq6NY/ovz3KsSZaABySO/f95BqGvptskZbJrMtC0pG4cSMcUmFWLHz5ZJHYApyvvJSA8V+jKA2QAghHiHiKoAjAHQWBQJmYyRoaxB8tQYhik/2DAoI4Tjr0SqIW6VhnSNynShRHHzuNZuI1HYqkoUCWvPFwSpI1111nRrW5AcA+f4TElfc7TDHgMm36TyYhiT9wFMJ6L9AOyAkVz8Occx2wCcDeBBIjoEQBUAjhViGIYpERxKVEZIhTqTkCFjXPq2Xo1hIA0IqTPL3ARn8nEmmo8ueVjK75ljUGAli+0CpmDwswUAEELEAVwFYC6ANTCqD60iohuI6L/Mw74H4KtEtBzAIwD+RwSppcwwDMMUBPYYlBH+oUQuHoOAVYmkYSBDIeR7ZyiRc6Xf82dcoyTJ4z1zDLLUroKO4s7HTL4hx18GEEK8ACOpWN32c+X1agAnF1suhmEYRg97DMoIqVDrqgwBXp6EYKFECeHwGAi9YZBNKJFOHs8+Bjk2OPODDQMm71gJ8/xsMQOfTU0d2NPZW2oxGIbJM2wYlBHSHpB///DyOnywfa+1323lXueZ1xoGCamwk3keF8PAofcEKTtqk8ca5+UxkH8Lo2RxKBGTb/iRYgYTXX0JzF3VUGoxGIbJMxxKVEY4cwxuf20jbn9tI84+eB9jv0vjM10vs3jS3WMg9XU5Lr1cafBQIp3y/+cvHIsHFnyE/cYMdR9ojfc9xMbDXz0BTyzZjtpK70ebPQbu/Oq/DsXw6mipxSg7+JFiBgPc2ZthBjZsGJQRUv92DRly6VmmO15nLKSSj+0egzTDIIPEYN0x08fV4qZLj/Ael2Xy8eGTh+PwycN9jwuxy8CVK0+aVmoRyhp+shiGYZhypeChREQ0m4jWEdFGIrqu0OcbyMjcgqQQtvAgqURnkmOg2xZ3VCVKuFQlsjotB5A563KjHK/NlBm8ksowDMOUOwU1DIgoDOBOABcAmAngciKaWchzDmTUHAOdDeBarUjjHdAlMCcdHgNnVSJZ+9+p/njnGBS2uhDD9BfYhmUYhmHKnUJ7DI4DsFEIsVkI0QfgUQAXF/icAxaZQyCECOwFcNuuK2wUt1oSp8YRAVHTYxANm12FHU+Nd46B+z5PWMliygx+ZBmGYZhyp9CGwSQA25X3deY2JgtS5UqdYTwyJ0A/TtfHQLtNk2MQIkLE9BRIA0F6AQqpCFnnYG2LKTP4mWUYhmHKlUIbBl7VKo0DiOYQ0WIiWtzU1FRgccqbpFKVKBOPga5cqS6UKNXgTL4HwkQImy4CmWsg9xcyx8Aaz+uwTJnABgHDMAxT7hTaMKgDMEV5PxnATvUAIcS9QohZQvz/9u48Xq66vv/46zPL3bLcS8INBJIQwIBEtGJTBLVa3ADbn1SrPsDWWmtLN+uvtbVCrfys+9bWWnHhp7bWh4q45wdI2ESlQiTsJBAIW3KzkISbe5Pcbbbv7485Z+6ZmTN3nzlzZt7PxyOPzJxz7sx3zpnlfM7n+/l+3Yb+/v46Nyfe3LQ1BrW6EoUtm2HGIAGpZGXGYObmXGNQ5wnORBZaqSh/JhGzSMyMZfJk8wV9J4u0uHoHBncB68zsZDPrAC4GNtb5OVuWfzLvnAs9+ajZlWiK7EDYsuCoRMlAVyI/QCjN8DqDNs95VKKK/0VEJDpnXHEDb/nyHVE3Q0TqrK7zGDjncmb2LmATkAS+5pzbWs/nbGWToxKVdyUay+ZKy8P/rnp52KaTk55NjkqUSBipUvHxXDIGc6OMgcSN3qvS6u7dORR1E0Skzuo+wZlz7nrg+no/T3vwawzK+/cfHc+Vlof+VcjysOJjP4DwMwYF50gmJjMG/onPbOYGm+s8BJPFxzrbknjRW1Za2Y1b90XdBBGpI818HCP+Bf3KjMGRiWJgUKvGYKZdiXL5ycDjuR/4CYUCLOlKleYvSMxh0rF5Zwzm+PcijaZCeWkHn7t1R9RNEJE6UmAQI5M1BuACk5b5J/Q1JzgL7UpUewjTiWye8WzxCcysNH/BXPr9z7fGYKHd8nevYOfgaJ0eXdqZMgUiIhJ39S4+lgXkn8oXnCtNdubfD/5faaoRiMKWBbdPJigNVzqZMZh5m+fdFWiBT7ZO7V/MeaevWNgHFUHZLWktw2NZvvKLJ2pmokWkNSljECPl8xhMLg8OYxr6d4WQZWHFx17mIRdYmQxmDEoFwQ04BSqNfKTTLRGRRvvAjx5i4/17WL9yKS95zrFRN0dEGkQZgxgpn8dg5hOczXQytGy+GEEEswmJhJVqDGwWw5TOV6nbkuICiQm9V6WVHB7PAvDWr2zmrqcGI26NiDSKAoMYCc5jELziH1w+1d8FhRYfF0IyBoFRiRKljMHs2z5bOsmSuFF2S1rVNXftiroJItIgCgxipCxjEKgxmLYr0QwnQ/MzBkHFCc6Kb5NlizoAeM6KxeXtYuH7oOokS+JGway0ghse2sfay65jxBvtTkTai2oMYiRYY+BCMga1uhKFZQfCtvVrDIISCSPp1RiceWIvl778FM4+edms2z5bOsmSuFJQK3H2uVseA+CpZzV6m0g7UmAQI8HMQPDEvjBtxiAkMAjtSlSdMUgYpL2MQdKM31zXX7WNToRERERE4k9diWLE77LjamQMatUYzHTm40xYxsAmi48TNaY8rk9XIpF40Szd0kr0bhZpTwoMYqTWzMc5rzag1nDToV2JQmc+DhnXFEh5XYmSFSc+9TwP0jmWxI3esiIiEncKDGKkVEtQKA8CwiYmAxgcyXgjGFUHAcNjWY6MZxkey5aWhdUYjGXzVaMS+eo5742uvkrc6C0rIiJxpxqDGCmb+TgYGJSKjyeX7R4a46WfuJX3nn86/Ys7qx7r63c8zdfveLpsWTakxqCvO10alahWVyLVGIhIGDO7APh3IAl8xTn3iZBt3gJ8kOJX3P3Oubc2tJEiIlKijEGMuFItQUXxcaF8PcDeoTEAbn1kf2g9QZjKjMHlFz6XL/7Br092JWpgjYFI3Cg8LmdmSeBK4EJgPXCJma2v2GYdcDnwUufc84C/aXhD25xzjs/f+hgHjkxE3RQRaQIKDGLEzwjkXfmpuD+aUDBY8G/V6koUprLG4FVnrOCEvu5SV6JG1hiUnqP+TyGyINT9rcrZwA7n3BPOuQxwNXBRxTZ/ClzpnDsE4Jzb3+A2tr37B4b5zI2P8rffuS/qpohIE1BgECPOTdYSzGq40lorKmQrtvO7EE03KlE9KRchcaG4oMqJQHDK3AFvWdBpwGlm9j9mdqfX9UgaKO9dWBrNlE9oFnw/f/fugUY2SUQipBqDGAkGAGFJgGCw4H+nm1nNgKFSNleeMUinioFBOunPYzCr5oq0JQUIJWF7ovLbKAWsA34LWAX8wszOdM4NlT2Q2aXApQBr1qxZ+JaKiAigjEGsBOcrCJuzoFaPoRl3JaqIINIVtQVRZAxE4kKfjioDwOrA/VXAnpBtfuycyzrnngS2UwwUyjjnrnLObXDObejvr55kURaOsrQi7U2BQQwVXHi5b60AIGwegzDZihoDf8bjJV0putNJVizpKlu/fmUvAC99zrEzenyRlualCuo5jG/M3AWsM7OTzawDuBjYWLHNj4DzAMzsWIpdi55oaCsllEabE2lP6koUI8F5DMKCgLDiY5j5iUrlqER+V6IlXWnu/MdXsaSz/O3y/FW93H/Fa+ntSc/sCURamE6jyjnncmb2LmATxeFKv+ac22pmHwK2OOc2eutea2bbgDzwXufcs9G1WkSkvSkwiJGymY9DJikOJgaCBce1Mgk9HUlGM/nS/cp5DNKBooLe7vCT/3oHBTrZkrhQbUE159z1wPUVy64I3HbAe7x/0kT0fhZpT+pKFCN+ByLnwucOCJsNGag5j0FHqvzwV2UMEnp7iMyWTqgkjtQDTkRAgUGsTI5K5EK7BwULkoOFxLW6EnVWBgaBjEEyYU1RbKwfK4kL9cmWeKoxcaW+fEXakroSxUhwHoPphiv1MwYPDAyxtCv8MFdmDDK5yb9Pa2xSkVlRpkBagd7GIu1NGYMY8c/7natVfDx5288YZPOOn24/EPp4nalk2f1gxsCfu0BEZkYnVCIiEnc6+4uRQiBjMN2oRPnASf6rzziOL/z+i6q270jWrjFQYCAiIiLSXnT2FyNlMx+HrA/GCtnASX5PR7KqngCgM12+LDiPgboSicyOuhJJK9H7WaQ9KTCIEf9Uv1Bj5uOwGgMoXv1PhHzLVxcfN1/GQL9NEhcqPpY42Tc8zoev3VZzOGsRaU8qPo4RPxhwLnzEiLDKHaKsAAAgAElEQVQaA4COlIVe/amqMQhkDCq7GYnINBQXSIz8/Xfv5/YdB1m2qKO07M++sYVtew9H2CoRiZrO/mKkvMag9noorzGYacYg2P0opa5EIiIt6bM3P8rtOw6WLXMONm19JqIWiUizUGAQI/65fr4QXnxcax6DVCI8MKgcrvTIRLbsb5qBktwSFwqlJS4+e/Njpdv+T8N9u4bKttH7WaQ9NcfZn8xIeY1ByPoaMx+nU0bYXGWVXYn2Do1P/o0yBiKzYqrWlBayZ3h8+o1EpOUoMIgRPyOQL0xffBwcerQjmQi9/FOZMQhmGZJNMOuxSJzoEyNx9PDeI1E3QUSaiAKDGJmsMQjvYlOokTGo1ZUobAjT4N80A51sSVwoYSBx8NXbnyy7///u3xNRS0SkGTXH2Z/MiJ8QKMyyxqDYlSgkMEhXH34/UaCMgYhIaxkcyfDha7dF3QwRaWIKDGLEDwbyNWoMyjMG5UOPhtYYhAxJekJfN6BRiURmSxkDaXaas0BEpqN5DGLE/06vNSpRoeaoRBZaGNmZTlYtW76og4FDY/POGHz8jc9n2x6Nhy3tQxOciYhI3CkwiJHpRiWqVWOQToVnDMImMUt5y1LzDAwuOXvNvP7ep+tbEhfKGIiISNypK1GMFIKjEoWcMtesMag1wVlIjYE/TKlqDETmRsOWSrPSO1NEpqOMQYwERyUKlBBUrQfI5YMzH4cXH4dlDNKljEFzxIz6IRMRmbt//OGDPLR7mAcGhtn8j6+Kujki0uQUGMRI2QRmYTUGgWChMmMQdhEzLGPgdyFSxkBkdvxMQdgcIyL1kskVGDg0yin9i0PXf2vzztLtkYlco5olIjHVHJeFZUaC5xvBCcxK6wPdi/Iz6ErUlaouPi7VGGhUIpFZ0SdGGu07d+3ktH/6Ca/8l59x4MhE1M0RkRagwCBGykcdqu5LFCw+Ls8YGEu6qpNDlTMf+9vC/IuPRUSkvj69aXvp9uHx7LTbq/5FRKajwCBGps0YBDbI58szBsct7aravjMkY5D0aguSTVJjIBIX/jmXTr5ERCSudPYXI8GMQbCr0OT6yduVNQZh2QFlDEQWjuYxkCj5Pw/OOb70s8cZHq3OIOgdKiLTUWAQI8ET/2xoV6Jg4FA+KlGYzrDAoJQx0E+IyGwoUSDN4H92PMsnfvII7//Rg1E3RURiSIFBrMw9YxAmLGOQUsZAZE70ialmZheY2XYz22Fml02x3ZvMzJnZhka2rxVlvaGqj4aMQPSNO59udHNEJGYUGMRIYZoag0Kh9qhEYUIzBt62SY1KJDIn+uQUmVkSuBK4EFgPXGJm60O2WwK8G9jc2Ba2mvLfhLBRc796+5MNaouIxJUCgxjJF1ypW1DYqETZwKRmYRmDW/7uFXzqTS8oLe9MhwxXmlDGQGQu/K5EmsWg5Gxgh3PuCedcBrgauChkuw8DnwLGG9m4lqWvbhGZBwUGMVIouNJJfi6kK1EmEBjkK4YrBTi1fzFv2bC6tDxs5mN/HgONSiQyWzojq3AisCtwf8BbVmJmZwGrnXPXNrJhrUjz6onIQpjX2Z+ZvdnMtppZobJvqJld7vUr3W5m58+vmQLF2Y79K/lhXYkyuakzBpXCAgN/UTLiSkrNHitxo+LjKmF7pPTBNrME8G/A3037QGaXmtkWM9ty4MCBBWxi6/rZowc007GIzNp8Lws/BLwR+HlwodeP9GLgecAFwBe8/qYyDwXnSgXDoRmDYGAQyB7U6hY0VR2BehKJzI0+OiUDwOrA/VXAnsD9JcCZwG1m9hRwDrAxrADZOXeVc26Dc25Df39/HZscf8H3356hscjaISLxNK/AwDn3sHNue8iqi4CrnXMTzrkngR0U+5vKPBQKk1f/gyf+vkyNGoNaEy6FBQz+nyUUGYjMij4xVe4C1pnZyWbWQfFi0UZ/pXNu2Dl3rHNurXNuLXAn8Hrn3JZomhtvr/m3n3PgyETUzRCRmKtXR/Jp+5bK7OWdKw0nGjZc6dBolrWXXcfgSKZsfWWXoSVdKQASIQGDPxdC2LpG0uyxEjd6z5ZzzuWAdwGbgIeBa5xzW83sQ2b2+mhb15puefiZqJsgIjGXmm4DM7sZOD5k1fudcz+u9Wchy0I7jZvZpcClAGvWrJmuOW0tP03xse+h3cNkcgV+bXUf737lc+jtSZet3/Q3L+fJgyOhGQO/a3+NsgQRqUFhQTXn3PXA9RXLrqix7W81ok2tLhig/uHXfhVhS0QkjqYNDJxzr57D407XtzT4+FcBVwFs2LBBFac1+HMUdEzRlch3aDRDJldg7bE9vOqM46rWn9DXzQl93VM+T9QZA5G40UdGms3eYY0AKyKzU6/rwhuBi82s08xOBtYBunQxD3nvUn6tjEFwFuNDIxky+QIdqdnXe/sPq24RItIunHN88oZH2N0Cxbr65haR+ZjvcKVvMLMB4FzgOjPbBOCc2wpcA2wDbgD+yjmXn29j25lfM+DXGFQOV9oVCAz2HZ4gkyuEDkc6nckag7m2dGFouFKJG8XS8bV1z2G+eNvjvOtb90TdlHnZf2SCK3+6I+pmiEiMTduVaCrOuR8CP6yx7qPAR+fz+ALj2Txd6WSp77+fMchWzHzclU5yeLw4ZvV9uw4xkSuUZRFmyjVJ8bGvSZohMi3TtdrY8i+IhM0PEyf/etOjUTdBRGJOJaZNbPfQGM//4CYeHBgudSXyswCVoxJ1picP5Z1PDHLw6ASdcwgMSsOVNsk5jhIHEhtN8pmR2dP3jIhIkQKDJrZ3aIxs3rFneKy6K1FFYLCoozr5M5OMwY1/+3J++JcvKd33r5xFXWMQ9fOLzJXeuvFy2/b9DBwq1hbo2IlIu5tXVyKpr7FssSyjUHCl0YL8rkT5ipR3T0d1ofFMagxOO24JTz87UrrfLPMYiMSNPjHx9Ef/eVfUTZiTu54a5ODRTNTNEJEWo4xBExvPFusIcgVX1ZUoV1FjsKizGOMF5yaYaVeiYN9o/2GbpSuRSFwoyxV/cTqCb/7SHVE3QURakAKDJuZnDPJlGYPwrkR+xiBhVjqpn2nxcfB8Jt8kGQONSiRxE6eTShERkTAKDJrYeGYyMPBP2FOlCc7CawwSCUh6kcFcAoNSV6ImSRnoIqyIiIhIY6jGoImN5wIZg8rhSitmPu7pnMwYOANwMw4MgtkB12SjEonEhYLYFhCTgzg4otoCEakPZQya2JifMXCTXYk6vK5ElcOV+hmDpFnpRH+mE5wFAwM/Y5BUZCAyK5rHQOpl655hfvn4wdL93/viLyNsjYi0MgUGTays+LhQ0ZWoIjDo9moMzJhXjYH/sM1SSKlSA4mLJvnIyDzUOoSjmeLkkbsGR3npJ25l7/BY4xoF/Pbnbuet/3dz6f6TB0em2FpEZO4UGDSx4HClfo1BusYEZ5M1BpMZgxmPShRWYxDxSU6zBCYiM6V3bPxkcoVpt3lo9zDrr9jETx7cyzc372T30Bg/vHd3A1oHE7k8RydyZcv2Hx5vyHOLSHtSjUETG/cCg1yhuitRrRqDpBnOiwc6U9VzG4QJdoFwGpVIZF4U08bH+77/QNl9M9gzNEY2X2DVMT0kE8YDA8MA/PyxAyztTje0fW+48pds23u4bNnZH7uloW0QkfaijEHExrN57tl5qOY6gHyhUJUxqOxK5AcBNofhSoPZgWabx0AnWRIb3ntVMW183Lh1X9Wyl3ziVl7x6dv48LXbav5do+pJKoMCEZF6U2AQsQ/86CHe+IVfsmtwtGrdZGAwecKeqtGVqM+7kvXWF6+Zw3Clkz9y5595HACnH790Fq9CRFR83Fque3Bv9cKIg74Jb6Q6EZF6UVeiiPlXhIZGs6xeVr5uLJAxKLjyCc4quxIt7krx6EcuJJ00vv2rncBsRiWavP2Gs1bxuuevnHE3JBEpUnYr/oKH0M/8jFT08YfojvXp/3RDNE8sIm1DGYOI+RmAbKG6CM4flShfmMwQ+FmAfMGV/Tilk0ZHKoGZkbS5Zwxg5rUJIlJNAUJ8zGSQg49e/zAARyfyUScMRETqToFBxFKJ8HkJoDxjUJr5ODFZYxAsEPaXw/yGK2026q8tcdHEHyOZoXt2DpVuHzw6UbYumyuUBkXQsRaRVqXAIGJ+YFDZNQhgIls9wZnflSiXL5T9OKWSk/f8q2DpxOwnOBORudEQu/FTORRoJb/OC4oXUJybvC0i0ooUGETMH2UobDztscBwpZVdiabMGHg3CzO83N7Mv3H6AZa40Hu1fajQXERalQKDiPlX+v16gqBSjUG+eoKzXL68xiCYMXj3K9cBsGJp54zaoIyByPzpU9R6ghmFfMGVRip6RpOMiUiLUmAQMb8rUdgwdGOBrkT+xf9gTUJZ8XEgY/DmDat56hO/TU/HzAadUlwgIu1kPJvngxu3Trvdho/cXLp947Zn2DtcDAi+cvuTdWubiEiUFBhEzO8CNJapDgwm5zGY7EqUTk2OYlTWlSg597N7BQYi86fPUXz89x1P8V+/fCrqZoiINB0FBhGb7Eo0TWDgpQz8uQmcK++6kJrHVMXqSiSyEPQ5qmRmF5jZdjPbYWaXhax/j5ltM7MHzOwWMzupEe0KGeuh6QyNZqJugoi0IQUGEfNrBsYqagxy+QLZfDEYyBcmRyUKBgDlGYO5H0qdzojMn+LrcmaWBK4ELgTWA5eY2fqKze4FNjjnXgB8D/hUY1vZvF72yZ9G3QQRaUMKDCLmn9xXZgzGA6MUhXUlgvJRh+bTlUgZA5GFoxFrSs4GdjjnnnDOZYCrgYuCGzjnfuqcG/Xu3gmsanAbm9Z0Q6mKiNSDAoOI5b0ZjysDg2DNQb7gSkFARyAzMBLYZqZzFoRpxrjgd886keWLOnjLhtVRN0VkRprwYxS1E4FdgfsD3rJa3gn8pK4tEhGRKc1s2Bqpm5yXCajKGATuF+cxKN6ulRmYX/Fx853SrDqmh7s/8JqomyEyY834OYpY2A4JnVzFzP4A2AC8osb6S4FLAdasWbNQ7RMRkQrKGEQs59URjE0RGOTdZMagVpHxfIqPRWT+9AmsMgAEU36rgD2VG5nZq4H3A693zk2EPZBz7irn3Abn3Ib+/v66NDZKuXyBtZddxzc3Pw3AkwdHIm6RiLQrBQYRy5W6EpUXHwfv5/OTgUGtq5K6WikiTeYuYJ2ZnWxmHcDFwMbgBmZ2FvBlikHB/kY1bKTJ+u/7F4Y+fv0jAJz3mdsibI2ItDMFBhHL1sgYjFVkDPzi46QCAJGmpI9mOedcDngXsAl4GLjGObfVzD5kZq/3Nvs0sBj4rpndZ2Ybazzcgrl52zN8/qc76v00MzKWyZPLV49IJyISFdUYNMh/3PIYv3laPy9c3Ve2PD+DGoObtj3Doo4kAEl1GRKRmHDOXQ9cX7HsisDtVze4PXz9jqca+ZRTOuOKGzh+aRcXnHl8adm7r743whaJSLtTxqBB/uWmR/ndK/+nannWuzo0mgnPGPijEP3ovmLX3ETC+PNXnLrg7Xvny07mW3/y4gV/XBGRZvHDe3fzi8cORt2MMvsOj5fNwnz9g/uia4yItD0FBg3gT04Wxh+V6Oh4eZ9XP2OwqDNZtjxpxmUXPneBWwgf+J31vOQ5xy7444q0C1f7Yy5N4uG9h6NuQslYpnq2exGRqCkwaIBsoXafUb8/aeVkNpOBQXlvr3lMVyAiDaBag+bVTJM5nnHFDVE3QUSkik4zG8AfkjR0nZcxODyeLVvuj0rUmSo/RCo+Fmluyhw0sQX8+nzbVzdz366hhXtAijUQIiJRUmDQANkpRpnwg4ajE7myHwW/xqCy2LiZrniJyCR9NNvLLx47WFY39tHrtvHCD90YYYtEROZPoxLV2RU/fogzVi4NXXfVzx9n+zNHgOJVxrd99Vd8/Y/PJpmwUleiykAgoVGJRJqSLvY2v8paroUwMpHjnI/fwpE6PLaISKMpMKiz79y1i3NPXR667mPeZDa+23ccZHAkQ/+STp49mqG3O10aztTnZxCuvvQcBg6NsbK3i4NHQycLFZEIKHPQnLbtOcw3N+9c8Me99oE9CxYUjKggWUQipsCgjgoFx0SuwKGRzJTbdaYSTOSK3Y38bkd7h4sn/ZXzG/g1BuecEh5siIhItXqNSPS+7z9Yl8cVEYmCagzqyD/ZHxydOjA4pqejdNuvLdg7PM7K3q7SzMg+jUokIjJ7v3z82aibICLS9HSaWUf+1f5DI9kpt+vrSVf9zd7hcVb2dZOpKFzWqEQizUklBs3t+/cMRPr8zjk+e/Oj6vopIk1NgUEd+Vf/K+cogPJh6Xo6JicxG8/mGc/mGRzJsHJpV9WIRpWjFIlIc9EnVMJsefoQn735Mf7+u/dH3RQRkZpUY1BHlfUB5esmT/iD9cXf/tUuvnf3HQCs7OsmmysPDEwZA5GmpsyBhPEHkhidUIGxiDQvZQzqaGyKwODIxGT3opW9XVxy9hoAvnf3QNnyyhoDEWlOCtkFYO1l13Hpf2+pWu6/PxyOTK723DYiIlFSYFBHwaxApeDwdmPZPO946dqqbVb2dlXVGIiISHO7cdszVRljfw4a52DDR26KolkiItNSYFBHU3UlCk60M57N051OVm2zsre7Lu0SkYWn3J4E7RkaK7vvZwwKznFYk6GJSJNSYFBHUwYGE8HAoEBnuvpQdHdUBwsi0tzUpUigvB5sNJMrBY4KIEWkmbVE8fF4Ns9oJs+yRR3Tb9wAo5kcqUSiZo2Bc45nDo+X7tfKGIiISDz5A8hN5PKsv2ITL1zdB8C+4fEp/kpEJFotERi87/sPcN+uIX723vPq+jy3bd/Pe665n5//w3ks7izuupd98lZOWt7Dv7z5hbzmX3/Gx974fP762/dO+Tj/9KOH+ObmnaX7a5cvoqtGYLB6WTe7BsfoSiemrFkQEZHmYV7uaM9QMRC4b9cQUJyjRkSkWbVEYLC4M1VWzFsv9+0aYnAkw/7D4yzuXwzAwKExBg6N8fC+wxyZyHHdA3unfZwHdw9z2nGL+etXrmPZog5esKqXdDJBKmHkCo73XfBcXvf84wH4wV+8lJ2DI6w+poeBij6rIiLSnPyeROd95rZI2yEiMhutERh0pcqKeetlr3flx68PCA4556eHH91/ZPrHGR7nlaev4H/92glly7vSSY5O5HjVGSs4afkiAPqXdNK/pBOAFUu75v8iRERERERCzKv42Mw+bWaPmNkDZvZDM+sLrLvczHaY2XYzO3/+Ta1taVeaTL7ARK6+E8fs9eoC/OxEsE5g5+AoAE8/OzrlY2RyBQ4eneD43uqTfL87Udg6ERGJj4RmqReRGJrvqEQ3AWc6514APApcDmBm64GLgecBFwBfMLO6Vdf6/f3rnTXY63Xl8QODYF/R+73+o/nC1GNO7B0ewzk4oS8sMEiwuDPF0q70QjVZREQioLhAROJoXoGBc+5G55x/Nn4nsMq7fRFwtXNuwjn3JLADOHs+zzWVUmAwMXVgMDKRKw0hOjiSAeDZoxMcGc+Wsg3+8jB+d6HD41kOjWS4++lDpXV+YDAd/2/C5ijoTidZqWyBiEjsnfvxW/mzb1TPgCwi0swWssbgj4HveLdPpBgo+Aa8ZXWxpKv4MqYrQH7e/9nECb1dfO0dv8EFn/0FbzvnJL5x59MAvPjkZfz5K07lHf91F1dfeg7nnLK87G+PjGc54gUeH7l2G//wvQfK1o9kJrsxndjXze5AoXBwRKH3XHM/AGuW9VS1b9miDpZ2K1sgEkfOaYR6Kbdp6zNRN0FEZFamzRiY2c1m9lDIv4sC27wfyAHf9BeFPFTor6aZXWpmW8xsy4EDB+byGlg8w8AAYM/wONv3FQuE/aAAYPOTg/zy8YMA3LPzUNXfBceeDs5a+WevOKVq23XHLS7d/vl7z2NxZ/nJ/htfdCJrj11U9Xf/fvFZfOwNz5/2NYhIEzP1IRERkXiaNjBwzr3aOXdmyL8fA5jZ24HfAX7fTV4yGwBWBx5mFbCnxuNf5Zzb4Jzb0N/fP6cXscQ78Z6qK1FwBKHRzNRFymEX/vbUGHv69160iiWd5YmX045bUrq9ZnlPKaPhqxyNyHd8b1dpBCIRERERkUaa76hEFwDvA17vnAsOx7MRuNjMOs3sZGAd8Kv5PNdU/BPvoxPZmtsERxB68uBI6DZ+YHHw6ETVun3Dxa5BlRcDj+/tKo0i5K9bt2Jx2TaVgcEJIfUFItIi1KWoxMwu8Eam22Fml4Ws7zSz73jrN5vZ2sa3UkREfPMdlejzwBLgJjO7z8y+BOCc2wpcA2wDbgD+yjlXt7FEZ9KVaE+gz/89T1d3FQJ49JmjwOR8BeV/P45Z9Un90q40K/uKy1YfU6wbWBfIGMBkcbRPw5GKSKvzRqK7ErgQWA9c4o1YF/RO4JBz7jnAvwGfbGwrRUQkaF7Fx96Xea11HwU+Op/Hnyn/xPu+XUOs7A0v9try1GDp9gMDw6HbPOgtf2z/EW7aVv449+4aon9xJ73d6bLCYoCV3sRj61YsZufgKKuPKQ8eKgODpV0tMa+ciIRRjYHvbGCHc+4JADO7muKIddsC21wEfNC7/T3g82ZmTpXcItLGDo9nIxu6viXOULvSSY7pSfODe3bzg3t2T7t9Jl+YcvnjB0b40/+uHmbunFOW0ZVOsm1v8f6yRR0AnH78Evp60vz62mO4f2C4tPzsk5cB5SMQrV3eg+nEQaTl+EMQv+aMFRG3pGmcCOwK3B8AXlxrG+dczsyGgeXAweBGZnYpcCnAmjVr6tVeEZGmcPO2Z3jji1ZNv2EdWDNdmNmwYYPbsmVu4z7vPzLO/sPVtQFByxd3kMs7hseyrD12ETufHWX1sm4KBRgczTAykWPtsYt4qkYNwprlPSTN2HVolBP7ukkmjJ6OFLl8gaMTORZ1phidyNPbk+bweJbOVILOVJLxbJ5svsAzhyc4dnEHfT0dc3qNItLcDo1k6O1O12XWWzO72zm3YcEfuE7M7M3A+c65P/Huvw042zn314FttnrbDHj3H/e2ebbW4871dyKXLzDqzWOzuCNVOkbOOQquOJReo2Yrds6RLzjyzpEww4BkwjCz0rC3/gWkQsGRKzjGMnm6OhIkzTg6kSORMCayBXo6kqSTxV7BY97rSxgkzMjlHUcmsqQSCVJJK5W/+PP5HB7P0r+kk8NjudLvlD8Ahj9ZZ3c6CVYcwCOVSNCZSrDv8DhLulJk847BkQkSZhScI5t3nHzsIobHsgyOZFh1TDcP7z1CRyrBvuExTj9+Kccu7mD7viMkzFh33GLMjKVdKYbHsuTyjj1DYxy7pJOH9x7m8FiWfYfHWb+yl8cPHOXAkQnWLOvBDO+5eljalWYiVyCdTLC0O1WqIVy7fBFDY1mGx7I8uu8Iz45kyOYLHBrJsKQrxXnPXcHxS7sYzxUYnciVagz3H5lg3/A4D+weZs/QGC9f18/e4TEWd6Y4oa+bJV0phkazZPMFcgXHzsFRnjhQfM4lXSmO6Ukzli1wdCLLrsEx3rJhFcct7eKEvm56u9MMjmRY1Jlk96ExutJJnIOCc4xm8vR0JLnrqUOctaaP7nSSXYdG6UgmODSaYeDQGCctX8Sp/YvoShfPKx4/cJR9w+Oc0NdNLu84OpEjmy+UjnV3OslxvV3k8sX9MziSIZU0RibyHBnPsnxRJy9Y3cvQaJaxTJ77B4ZImNGdTrLl6UMcGc9ywZnHs6Qrxan9i1nUkWJoLMPtO54lYcVyquOWdjI4kuXU/kX0L+kkV3B0pRJM5ArsHR4nVyiQzRXfhyf0dnPS8p7SezXrnY/19aT52fYDpJLF96xZse3PjmRY2dvF6ccvZXBkgnyheKF1LJOjpzPFUwdHmMgV2D00xmvXH8doJk8mV2DP0BjjuTwn9nXTlU7yyL7ie/Dx/Uf5tVV99PakSSeNsUyBxV0pDhwZZ/WyHu58YpAVSzqZyBVIeO/5TK7A807sZcWSToZGM0zkCnSlk0xk8zx+YIRMvsCuwVF6OpKceWIvdz11iBP7unn8wFEGRzLk8gVGMnlWHdPNKf2LOW3FYk5a3sPOwVGGRrPc/fQhenvS/MbaZTjnGBrNcv7zjufV64+b03fLQvxOtExgICLSymIYGJwLfNA5d753/3IA59zHA9ts8ra5w8xSwD6gf6quRPqdEBEJtxC/E/MtPhYREQlzF7DOzE42sw7gYooj1gVtBN7u3X4TcKvqC0REotMSNQYiItJcvJqBdwGbgCTwNefcVjP7ELDFObcR+CrwDTPbAQxSDB5ERCQiCgxERKQunHPXA9dXLLsicHsceHOj2yUiIuHUlUhERERERBQYiIiIiIiIAgMREREREUGBgYiIiIiIoMBARERERERQYCAiIiIiIigwEBERERERwJppkkkzOwA8Pcc/PxY4uIDNiSvtB+0Dn/ZDa+2Dk5xz/VE3Imr6nZiVdnu90H6vWa+3tc329c77d6KpAoP5MLMtzrkNUbcjatoP2gc+7QftAynXbu+Hdnu90H6vWa+3tUXxetWVSEREREREFBiIiIiIiEhrBQZXRd2AJqH9oH3g037QPpBy7fZ+aLfXC+33mvV6W1vDX2/L1BiIiIiIiMjctVLGQERERERE5qglAgMzu8DMtpvZDjO7LOr21IuZfc3M9pvZQ4Fly8zsJjN7zPv/GG+5mdnnvH3ygJm9KLqWLywzW21mPzWzh81sq5n9b2952+wLM+sys1+Z2f3ePvhnb/nJZrbZ2wffMbMOb3mnd3+Ht35tlO1faGaWNLN7zexa735b7gepLU6/Ewv5HWdmb/e2f8zM3h5Y/utm9qD3N58zM5vqORr0uuf9OTazy73l283s/MDy0ONf6zka9Hr7zOx7ZvaId6zPbeVjbGZ/672fHzKzb1vxd6yljrEt0HnaQh3TqZ6jJudcrP8BSeBx4BSgA7gfWB91uxA93VoAAAUcSURBVOr0Wl8OvAh4KLDsU8Bl3u3LgE96t18H/AQw4Bxgc9TtX8D9sBJ4kXd7CfAosL6d9oX3WhZ7t9PAZu+1XQNc7C3/EvAX3u2/BL7k3b4Y+E7Ur2GB98d7gG8B13r323I/6F/N90esficW6jsOWAY84f1/jHf7GG/dr4Bzvb/5CXChtzz0ORr0uuf1Ofb20f1AJ3Cyd8yTUx3/Ws/RoNf7deBPvNsdQF+rHmPgROBJoDuw3/+o1Y4xC3CetpDHtNZzTPkaGvUBqONBOBfYFLh/OXB51O2q4+tdW/GG2w6s9G6vBLZ7t78MXBK2Xav9A34MvKZd9wXQA9wDvJjiRCgpb3npswFsAs71bqe87Szqti/Q618F3AK8ErjW+wJsu/2gf1O+R2L9OzHX7zjgEuDLgeVf9patBB4JLC9tV+s5GvAa5/05rjyu/na1jv9Uz9GA17uU4omyVSxvyWNMMTDYRfFkN+Ud4/Nb8Rgzz/O0hTymtZ5jqva3Qlci/83mG/CWtYvjnHN7Abz/V3jL22K/eOnFsyheMW+rfWHFtPt9wH7gJopXS4acczlvk+DrLO0Db/0wsLyxLa6bzwL/ABS8+8tpz/0gtcX2O2Ce33FTLR8IWc4Uz1FvC/E5nu1+mOo56u0U4ADwn1bsPvUVM1tEix5j59xu4DPATmAvxWN2N619jH1RHtNZf/e1QmBgIctcw1vRfFp+v5jZYuD7wN845w5PtWnIstjvC+dc3jn3QopX2s4GzgjbzPu/JfeBmf0OsN85d3dwccimLb0fZFqxPO4L8B032+WRWMDPcZz2Q4pil5MvOufOAkYodgGpJU6vrYrX5/0iit1/TgAWAReGbNpKx3g6jXgts/6bVggMBoDVgfurgD0RtSUKz5jZSgDv//3e8pbeL2aWpviD+U3n3A+8xW25L5xzQ8BtFPsP9plZylsVfJ2lfeCt7wUGG9vSungp8Hozewq4mmI3hM/SfvtBpha774AF+o6bavmqkOVTPUc9LdTneLb74eAUz1FvA8CAc26zd/97FAOFVj3GrwaedM4dcM5lgR8AL6G1j7EvymM66+++VggM7gLWeVXnHRSLVDZG3KZG2gi83bv9dop9Uf3lf+hVpJ8DDPtpprjzqvC/CjzsnPvXwKq22Rdm1m9mfd7tbopfug8DPwXe5G1WuQ/8ffMm4FbndTiMM+fc5c65Vc65tRQ/+7c6536fNtsPMq1Y/U4s4HfcJuC1ZnaMd8X2tRT7V+8FjpjZOd5z/SHhn5Hgc9TNAn6ONwIXW3FEm5OBdRSLNUOPv/c3tZ6jrpxz+4BdZna6t+hVwDZa9BhT7EJ0jpn1eO3xX2/LHuOAKI/p7M9/6lmA0ah/FKuuH6XYx/r9Ubenjq/z2xT75mUpRoHvpNh/7hbgMe//Zd62Blzp7ZMHgQ1Rt38B98PLKKbCHgDu8/69rp32BfAC4F5vHzwEXOEtP4Xil+QO4LtAp7e8y7u/w1t/StSvoQ775LeYHM2kbfeD/tV8f8Tmd2Ihv+OAP/be7zuAdwSWb/C+Ox4HPs/khKehz9HA1z6vzzHwfu81bccbsWWq41/rORr0Wl8IbPGO848ojkDTsscY+GfgEa9N36A4slBLHWMW6DxtoY7pVM9R659mPhYRERERkZboSiQiIiIiIvOkwEBERERERBQYiIiIiIiIAgMREREREUGBgYiIiIiIoMBARERERERQYCAiIiIiIigwEBERERER4P8DffsNRcY3MSMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_training(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()\n",
    "\n",
    "plot_training(i, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
